From 1539dbbd3211c6fe8645149f83b0413ccabd8af8 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Mon, 24 Mar 2025 18:59:21 -0700
Subject: [PATCH 1/3] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?=
 =?UTF-8?q?anges=20to=20main=20this=20commit=20is=20based=20on?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Created using spr 1.3.6-beta.1

[skip ci]
---
 clang-tools-extra/clang-query/Query.cpp       |    2 +-
 clang-tools-extra/clang-tidy/ClangTidy.cpp    |    6 +-
 .../cert/DontModifyStdNamespaceCheck.cpp      |   32 +-
 .../clangd/HeaderSourceSwitch.cpp             |    4 +-
 .../unittests/HeaderSourceSwitchTests.cpp     |   38 +
 clang-tools-extra/docs/ReleaseNotes.rst       |    6 -
 .../test/clang-doc/DR-131697.cpp              |   12 +-
 .../Inputs/basic-project/src/Calculator.cpp   |    8 +-
 .../Inputs/basic-project/src/Circle.cpp       |    5 +-
 .../Inputs/basic-project/src/Rectangle.cpp    |    6 +-
 clang-tools-extra/test/clang-doc/enum.cpp     |   68 +-
 .../test/clang-doc/namespace.cpp              |   49 +-
 .../test/clang-doc/single-file-public.cpp     |    4 +-
 .../test/clang-doc/templates.cpp              |   13 +-
 .../test/clang-doc/test-path-abs.cpp          |    1 -
 .../identifier-naming-anon-record-fields.cpp  |   22 +-
 .../clang-tidy/infrastructure/file-filter.cpp |    7 +
 .../infrastructure/system-headers.cpp         |    4 +-
 clang/bindings/python/clang/cindex.py         |   17 +
 .../python/tests/cindex/test_cursor.py        |   40 +
 clang/docs/ReleaseNotes.rst                   |   16 +-
 .../clang/ASTMatchers/ASTMatchFinder.h        |   33 +-
 clang/include/clang/Basic/Attr.td             |    4 +-
 .../clang/Basic/DiagnosticDriverKinds.td      |    4 -
 .../clang/Sema/HLSLExternalSemaSource.h       |   17 +-
 clang/lib/ASTMatchers/ASTMatchFinder.cpp      |   34 +-
 clang/lib/CodeGen/CGStmt.cpp                  |   13 +
 clang/lib/Driver/ToolChains/CommonArgs.cpp    |    6 -
 .../lib/Headers/hlsl/hlsl_alias_intrinsics.h  |   48 +-
 clang/lib/Sema/CMakeLists.txt                 |    1 +
 clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp |  781 +++++++++++
 clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h   |  102 ++
 clang/lib/Sema/HLSLExternalSemaSource.cpp     |  749 +---------
 clang/test/AST/HLSL/HLSLControlFlowHint.hlsl  |   59 +
 clang/test/Driver/hip-code-object-version.hip |    2 -
 clang/test/Misc/warning-flags.c               |    3 +-
 .../ASTMatchers/ASTMatchersInternalTest.cpp   |    2 +-
 compiler-rt/lib/asan/asan_shadow_setup.cpp    |   14 +-
 .../lib/tsan/rtl/tsan_interceptors_posix.cpp  |   14 +-
 compiler-rt/test/tsan/cxa_guard_acquire.cpp   |   76 +-
 flang/cmake/modules/FlangCommon.cmake         |   40 +-
 flang/include/flang/Common/api-attrs.h        |   17 +
 flang/include/flang/Evaluate/common.h         |    1 +
 flang/include/flang/Evaluate/traverse.h       |    3 -
 flang/include/flang/Evaluate/variable.h       |    1 -
 flang/include/flang/Semantics/dump-expr.h     |    5 -
 flang/lib/Lower/IterationSpace.cpp            |    4 -
 .../Optimizer/Transforms/CUFOpConversion.cpp  |    7 +-
 flang/test/Fir/CUDA/cuda-launch.fir           |    4 +
 libc/src/__support/float_to_string.h          |    6 +-
 libc/test/UnitTest/ZxTest.h                   |    8 -
 libc/test/src/sys/epoll/linux/CMakeLists.txt  |    6 +
 .../sys/epoll/linux/epoll_create1_test.cpp    |    7 +-
 .../src/sys/epoll/linux/epoll_create_test.cpp |    7 +-
 .../src/sys/epoll/linux/epoll_ctl_test.cpp    |    5 +-
 .../src/sys/epoll/linux/epoll_pwait2_test.cpp |    5 +-
 .../src/sys/epoll/linux/epoll_pwait_test.cpp  |    6 +-
 .../src/sys/epoll/linux/epoll_wait_test.cpp   |    5 +-
 llvm/docs/ReleaseNotes.md                     |    3 +
 llvm/include/llvm/BinaryFormat/DXContainer.h  |    2 +-
 llvm/include/llvm/Support/Mustache.h          |  128 ++
 llvm/lib/Support/CMakeLists.txt               |    1 +
 llvm/lib/Support/Mustache.cpp                 |  763 ++++++++++
 .../lib/Target/DirectX/DXContainerGlobals.cpp |   94 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  395 +++++-
 llvm/lib/Transforms/Vectorize/VPlan.h         |    9 +-
 llvm/lib/Transforms/Vectorize/VPlanValue.h    |    1 -
 .../test/CodeGen/AArch64/complex-int-to-fp.ll |  155 ++-
 .../ContainerData/PSVResources-order.ll       |   26 +
 .../DirectX/ContainerData/PSVResources.ll     |   11 +
 .../CodeGen/DirectX/HLSLControlFlowHint.ll    |  131 ++
 .../SPIRV/structurizer/HLSLControlFlowHint.ll |  125 +-
 llvm/test/TableGen/AcquireAtCycle.td          |   12 +-
 .../SLPVectorizer/AArch64/vec3-base.ll        |    8 +-
 ...reversed-strided-node-with-external-ptr.ll |    7 +-
 .../SLPVectorizer/RISCV/vec3-base.ll          |    8 +-
 .../SLPVectorizer/X86/barriercall.ll          |    4 +-
 .../X86/bottom-to-top-reorder.ll              |   11 +-
 .../buildvector-postpone-for-dependency.ll    |    8 +-
 .../SLPVectorizer/X86/bv-shuffle-mask.ll      |    4 +-
 .../X86/extract-scalar-from-undef.ll          |   28 +-
 .../SLPVectorizer/X86/extractcost.ll          |    4 +-
 ...gathered-delayed-nodes-with-reused-user.ll |   34 +-
 .../X86/minbitwidth-drop-wrapping-flags.ll    |    4 +-
 .../X86/multi-extracts-bv-combined.ll         |    6 +-
 .../non-scheduled-inst-reused-as-last-inst.ll |   44 +-
 .../SLPVectorizer/X86/propagate_ir_flags.ll   |   12 +-
 .../reduced-val-vectorized-in-transform.ll    |    6 +-
 .../X86/reorder_diamond_match.ll              |    4 +-
 .../X86/shuffle-mask-emission.ll              |    8 +-
 .../Transforms/SLPVectorizer/X86/vec3-base.ll |   19 +-
 .../X86/vect_copyable_in_binops.ll            |    8 +-
 .../alternate-opcode-sindle-bv.ll             |   35 +-
 .../Transforms/SLPVectorizer/isOpcodeOrAlt.ll |   36 +
 .../resized-alt-shuffle-after-minbw.ll        |    4 +-
 .../SLPVectorizer/shuffle-mask-resized.ll     |    4 +-
 llvm/unittests/Support/CMakeLists.txt         |    1 +
 llvm/unittests/Support/MustacheTest.cpp       | 1226 +++++++++++++++++
 llvm/utils/TableGen/SubtargetEmitter.cpp      |    2 +-
 .../gn/secondary/clang/lib/Sema/BUILD.gn      |    1 +
 mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td  |    4 +-
 .../mlir/Dialect/Tosa/IR/TosaShapeOps.td      |    5 +
 .../Dialect/Tosa/Transforms/TosaFolders.cpp   |   53 +
 mlir/test/Dialect/Tosa/availability.mlir      |    4 +-
 mlir/test/Dialect/Tosa/constant-op-fold.mlir  |    8 +-
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |   28 +-
 .../amdgpu/utils/UtilitiesRTL.h               |    9 +-
 .../plugins-nextgen/common/src/Utils/ELF.cpp  |    5 +-
 .../libc/test/src/sys/epoll/BUILD.bazel       |   10 +-
 109 files changed, 4572 insertions(+), 1405 deletions(-)
 create mode 100644 clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
 create mode 100644 clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
 create mode 100644 llvm/include/llvm/Support/Mustache.h
 create mode 100644 llvm/lib/Support/Mustache.cpp
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/PSVResources-order.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll
 create mode 100644 llvm/unittests/Support/MustacheTest.cpp

diff --git a/clang-tools-extra/clang-query/Query.cpp b/clang-tools-extra/clang-query/Query.cpp
index 091713600686e..382aa5d6fe25e 100644
--- a/clang-tools-extra/clang-query/Query.cpp
+++ b/clang-tools-extra/clang-query/Query.cpp
@@ -114,7 +114,7 @@ bool MatchQuery::run(llvm::raw_ostream &OS, QuerySession &QS) const {
     Profiler.emplace();
 
   for (auto &AST : QS.ASTs) {
-    ast_matchers::MatchFinderOptions FinderOptions;
+    ast_matchers::MatchFinder::MatchFinderOptions FinderOptions;
     std::optional<llvm::StringMap<llvm::TimeRecord>> Records;
     if (QS.EnableProfile) {
       Records.emplace();
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index d99847a82d168..733a53a0f5dcc 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -420,7 +420,7 @@ ClangTidyASTConsumerFactory::createASTConsumer(
   std::vector<std::unique_ptr<ClangTidyCheck>> Checks =
       CheckFactories->createChecksForLanguage(&Context);
 
-  ast_matchers::MatchFinderOptions FinderOptions;
+  ast_matchers::MatchFinder::MatchFinderOptions FinderOptions;
 
   std::unique_ptr<ClangTidyProfiling> Profiling;
   if (Context.getEnableProfiling()) {
@@ -429,10 +429,6 @@ ClangTidyASTConsumerFactory::createASTConsumer(
     FinderOptions.CheckProfiling.emplace(Profiling->Records);
   }
 
-  // Avoid processing system headers, unless the user explicitly requests it
-  if (!Context.getOptions().SystemHeaders.value_or(false))
-    FinderOptions.SkipSystemHeaders = true;
-
   std::unique_ptr<ast_matchers::MatchFinder> Finder(
       new ast_matchers::MatchFinder(std::move(FinderOptions)));
 
diff --git a/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp b/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp
index 2dff4c0e53b8c..bc4970825b4ca 100644
--- a/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/DontModifyStdNamespaceCheck.cpp
@@ -35,30 +35,6 @@ AST_POLYMORPHIC_MATCHER_P(
                              Builder) != Args.end();
 }
 
-bool isStdOrPosixImpl(const DeclContext *Ctx) {
-  if (!Ctx->isNamespace())
-    return false;
-
-  const auto *ND = cast<NamespaceDecl>(Ctx);
-  if (ND->isInline()) {
-    return isStdOrPosixImpl(ND->getParent());
-  }
-
-  if (!ND->getParent()->getRedeclContext()->isTranslationUnit())
-    return false;
-
-  const IdentifierInfo *II = ND->getIdentifier();
-  return II && (II->isStr("std") || II->isStr("posix"));
-}
-
-AST_MATCHER(Decl, isInStdOrPosixNS) {
-  for (const auto *Ctx = Node.getDeclContext(); Ctx; Ctx = Ctx->getParent()) {
-    if (isStdOrPosixImpl(Ctx))
-      return true;
-  }
-  return false;
-}
-
 } // namespace
 
 namespace clang::tidy::cert {
@@ -66,10 +42,12 @@ namespace clang::tidy::cert {
 void DontModifyStdNamespaceCheck::registerMatchers(MatchFinder *Finder) {
   auto HasStdParent =
       hasDeclContext(namespaceDecl(hasAnyName("std", "posix"),
-                                   unless(hasDeclContext(namespaceDecl())))
+                                   unless(hasParent(namespaceDecl())))
                          .bind("nmspc"));
-  auto UserDefinedType = qualType(hasUnqualifiedDesugaredType(
-      tagType(unless(hasDeclaration(tagDecl(isInStdOrPosixNS()))))));
+  auto UserDefinedType = qualType(
+      hasUnqualifiedDesugaredType(tagType(unless(hasDeclaration(tagDecl(
+          hasAncestor(namespaceDecl(hasAnyName("std", "posix"),
+                                    unless(hasParent(namespaceDecl()))))))))));
   auto HasNoProgramDefinedTemplateArgument = unless(
       hasAnyTemplateArgumentIncludingPack(refersToType(UserDefinedType)));
   auto InsideStdClassOrClassTemplateSpecialization = hasDeclContext(
diff --git a/clang-tools-extra/clangd/HeaderSourceSwitch.cpp b/clang-tools-extra/clangd/HeaderSourceSwitch.cpp
index 2351858cc6297..d54c3668570eb 100644
--- a/clang-tools-extra/clangd/HeaderSourceSwitch.cpp
+++ b/clang-tools-extra/clangd/HeaderSourceSwitch.cpp
@@ -22,7 +22,9 @@ std::optional<Path> getCorrespondingHeaderOrSource(
     PathRef OriginalFile, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
   llvm::StringRef SourceExtensions[] = {".cpp", ".c", ".cc", ".cxx",
                                         ".c++", ".m", ".mm"};
-  llvm::StringRef HeaderExtensions[] = {".h", ".hh", ".hpp", ".hxx", ".inc"};
+  llvm::StringRef HeaderExtensions[] = {".h",    ".hh",   ".hpp", ".hxx",
+                                        ".inc",  ".cppm", ".ccm", ".cxxm",
+                                        ".c++m", ".ixx"};
 
   llvm::StringRef PathExt = llvm::sys::path::extension(OriginalFile);
 
diff --git a/clang-tools-extra/clangd/unittests/HeaderSourceSwitchTests.cpp b/clang-tools-extra/clangd/unittests/HeaderSourceSwitchTests.cpp
index 6e280f5b085a9..e600207de458a 100644
--- a/clang-tools-extra/clangd/unittests/HeaderSourceSwitchTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HeaderSourceSwitchTests.cpp
@@ -76,6 +76,44 @@ TEST(HeaderSourceSwitchTest, FileHeuristic) {
   EXPECT_FALSE(PathResult.has_value());
 }
 
+TEST(HeaderSourceSwitchTest, ModuleInterfaces) {
+  MockFS FS;
+
+  auto FooCC = testPath("foo.cc");
+  auto FooCPPM = testPath("foo.cppm");
+  FS.Files[FooCC];
+  FS.Files[FooCPPM];
+  std::optional<Path> PathResult =
+      getCorrespondingHeaderOrSource(FooCC, FS.view(std::nullopt));
+  EXPECT_TRUE(PathResult.has_value());
+  ASSERT_EQ(*PathResult, FooCPPM);
+
+  auto Foo2CPP = testPath("foo2.cpp");
+  auto Foo2CCM = testPath("foo2.ccm");
+  FS.Files[Foo2CPP];
+  FS.Files[Foo2CCM];
+  PathResult = getCorrespondingHeaderOrSource(Foo2CPP, FS.view(std::nullopt));
+  EXPECT_TRUE(PathResult.has_value());
+  ASSERT_EQ(*PathResult, Foo2CCM);
+
+  auto Foo3CXX = testPath("foo3.cxx");
+  auto Foo3CXXM = testPath("foo3.cxxm");
+  FS.Files[Foo3CXX];
+  FS.Files[Foo3CXXM];
+  PathResult = getCorrespondingHeaderOrSource(Foo3CXX, FS.view(std::nullopt));
+  EXPECT_TRUE(PathResult.has_value());
+  ASSERT_EQ(*PathResult, Foo3CXXM);
+
+  auto Foo4CPLUSPLUS = testPath("foo4.c++");
+  auto Foo4CPLUSPLUSM = testPath("foo4.c++m");
+  FS.Files[Foo4CPLUSPLUS];
+  FS.Files[Foo4CPLUSPLUSM];
+  PathResult =
+      getCorrespondingHeaderOrSource(Foo4CPLUSPLUS, FS.view(std::nullopt));
+  EXPECT_TRUE(PathResult.has_value());
+  ASSERT_EQ(*PathResult, Foo4CPLUSPLUSM);
+}
+
 MATCHER_P(declNamed, Name, "") {
   if (const NamedDecl *ND = dyn_cast<NamedDecl>(arg))
     if (ND->getQualifiedNameAsString() == Name)
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index ed7da975f3de7..aa85105918ecf 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -91,12 +91,6 @@ Improvements to clang-query
 Improvements to clang-tidy
 --------------------------
 
-- :program:`clang-tidy` no longer processes declarations from system headers
-  by default, greatly improving performance. This behavior is disabled if the
-  `SystemHeaders` option is enabled.
-  Note: this may lead to false negatives; downstream users may need to adjust
-  their checks to preserve existing behavior.
-
 - Improved :program:`clang-tidy-diff.py` script. Add the `-warnings-as-errors`
   argument to treat warnings as errors.
 
diff --git a/clang-tools-extra/test/clang-doc/DR-131697.cpp b/clang-tools-extra/test/clang-doc/DR-131697.cpp
index 67bd55b897973..9025bbf910813 100644
--- a/clang-tools-extra/test/clang-doc/DR-131697.cpp
+++ b/clang-tools-extra/test/clang-doc/DR-131697.cpp
@@ -13,10 +13,8 @@ int main() {
 }
 
 //--- compile_commands.json
-[
-{
-  "directory": "foo",
-  "file":"main.cpp",
-  "command":"clang main.cpp -c"
-}
-]
+[{
+  "directory" : "foo",
+  "file" : "main.cpp",
+  "command" : "clang main.cpp -c"
+}]
diff --git a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Calculator.cpp b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Calculator.cpp
index 64f31dbf13d87..258ea22e46184 100644
--- a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Calculator.cpp
+++ b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Calculator.cpp
@@ -1,17 +1,17 @@
 #include "Calculator.h"
 
 int Calculator::add(int a, int b) {
-    return a + b;
+  return a + b;
 }
 
 int Calculator::subtract(int a, int b) {
-    return a - b;
+  return a - b;
 }
 
 int Calculator::multiply(int a, int b) {
-    return a * b;
+  return a * b;
 }
 
 double Calculator::divide(int a, int b) {
-    return static_cast<double>(a) / b;
+  return static_cast<double>(a) / b;
 }
diff --git a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp
index 3ddb2fd9ff563..811aac4989bd0 100644
--- a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp
+++ b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Circle.cpp
@@ -3,10 +3,9 @@
 Circle::Circle(double radius) : radius_(radius) {}
 
 double Circle::area() const {
-    return 3.141 * radius_ * radius_;
+  return 3.141 * radius_ * radius_;
 }
 
 double Circle::perimeter() const {
-    return 3.141 * radius_;
+  return 3.141 * radius_;
 }
-
diff --git a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Rectangle.cpp b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Rectangle.cpp
index 7ffc769157ebc..a2b253b750aa8 100644
--- a/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Rectangle.cpp
+++ b/clang-tools-extra/test/clang-doc/Inputs/basic-project/src/Rectangle.cpp
@@ -1,12 +1,12 @@
 #include "Rectangle.h"
 
 Rectangle::Rectangle(double width, double height)
-        : width_(width), height_(height) {}
+    : width_(width), height_(height) {}
 
 double Rectangle::area() const {
-    return width_ * height_;
+  return width_ * height_;
 }
 
 double Rectangle::perimeter() const {
-    return 2 * (width_ + height_);
+  return 2 * (width_ + height_);
 }
\ No newline at end of file
diff --git a/clang-tools-extra/test/clang-doc/enum.cpp b/clang-tools-extra/test/clang-doc/enum.cpp
index ef768e33b4566..b05d8e2029070 100644
--- a/clang-tools-extra/test/clang-doc/enum.cpp
+++ b/clang-tools-extra/test/clang-doc/enum.cpp
@@ -14,16 +14,15 @@
 // RUN: FileCheck %s < %t/Vehicles/index.md --check-prefix=MD-VEHICLES-LINE
 // RUN: FileCheck %s < %t/Vehicles/index.md --check-prefix=MD-VEHICLES
 
-
 /**
  * @brief For specifying RGB colors
  */
 enum Color {
-// MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
-// HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-  Red, ///< Comment 1
+  // MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
+  // HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
+  Red,   ///< Comment 1
   Green, ///< Comment 2
-  Blue ///< Comment 3
+  Blue   ///< Comment 3
 };
 
 // MD-INDEX: ## Enums
@@ -49,8 +48,8 @@ enum Color {
  * @brief Shape Types
  */
 enum class Shapes {
-// MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
-// HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
+  // MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
+  // HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
 
   /// Comment 1
   Circle,
@@ -77,22 +76,20 @@ enum class Shapes {
 // HTML-INDEX: <td>2</td>
 // HTML-INDEX: <p> Comment 3</p>
 
-
-
 class Animals {
-// MD-ANIMAL-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
-// HTML-ANIMAL-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
+  // MD-ANIMAL-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
+  // HTML-ANIMAL-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
 public:
-      /**
-       * @brief specify what animal the class is
-       */
-      enum AnimalType {
-// MD-ANIMAL-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
-// HTML-ANIMAL-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-          Dog, ///< Man's best friend
-          Cat, ///< Man's other best friend
-          Iguana ///< A lizard
-      };
+  /**
+   * @brief specify what animal the class is
+   */
+  enum AnimalType {
+    // MD-ANIMAL-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
+    // HTML-ANIMAL-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
+    Dog,   ///< Man's best friend
+    Cat,   ///< Man's other best friend
+    Iguana ///< A lizard
+  };
 };
 
 // HTML-ANIMAL: <h1>class Animals</h1>
@@ -108,7 +105,6 @@ class Animals {
 // HTML-ANIMAL: <td>2</td>
 // HTML-ANIMAL: <p> A lizard</p>
 
-
 // MD-ANIMAL: # class Animals
 // MD-ANIMAL: ## Enums
 // MD-ANIMAL: | enum AnimalType |
@@ -118,21 +114,20 @@ class Animals {
 // MD-ANIMAL: | Iguana |
 // MD-ANIMAL: **brief** specify what animal the class is
 
-
 namespace Vehicles {
-    /**
-     * @brief specify type of car
-     */
-    enum Car {
-// MD-VEHICLES-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
-// HTML-VEHICLES-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-
-       Sedan, ///< Comment 1
-       SUV, ///< Comment 2
-       Pickup, ///< Comment 3
-       Hatchback ///< Comment 4
-    };
-}
+/**
+ * @brief specify type of car
+ */
+enum Car {
+  // MD-VEHICLES-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
+  // HTML-VEHICLES-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
+
+  Sedan,    ///< Comment 1
+  SUV,      ///< Comment 2
+  Pickup,   ///< Comment 3
+  Hatchback ///< Comment 4
+};
+} // namespace Vehicles
 
 // MD-VEHICLES: # namespace Vehicles
 // MD-VEHICLES: ## Enums
@@ -159,7 +154,6 @@ namespace Vehicles {
 // HTML-VEHICLES: <td>3</td>
 // HTML-VEHICLES: <p> Comment 4</p>
 
-
 enum ColorUserSpecified {
   RedUserSpecified = 'A',
   GreenUserSpecified = 2,
diff --git a/clang-tools-extra/test/clang-doc/namespace.cpp b/clang-tools-extra/test/clang-doc/namespace.cpp
index db947985a2ba6..4fa11c78d37c0 100644
--- a/clang-tools-extra/test/clang-doc/namespace.cpp
+++ b/clang-tools-extra/test/clang-doc/namespace.cpp
@@ -39,22 +39,19 @@
 // RUN: FileCheck %s < %t/all_files.md -check-prefix=MD-ALL-FILES
 // RUN: FileCheck %s < %t/index.md -check-prefix=MD-INDEX
 
-
-
 // Anonymous Namespace
-namespace
-{
-    void anonFunction() {}
+namespace {
+void anonFunction() {}
 // MD-ANON-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-ANON-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
-    class AnonClass {};
+class AnonClass {};
 // MD-ANON-CLASS-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-ANON-CLASS-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
 // MD-ANON-CLASS: # class AnonClass
 // HTML-ANON-CLASS: <h1>class AnonClass</h1>
-}
+} // namespace
 
 // MD-ANON-INDEX: # namespace @nonymous_namespace
 // MD-ANON-INDEX:  Anonymous Namespace
@@ -72,16 +69,15 @@ namespace
 // HTML-ANON-INDEX: <h3 id="{{([0-9A-F]{40})}}">anonFunction</h3>
 // HTML-ANON-INDEX: <p>void anonFunction()</p>
 
-
 // Primary Namespace
 namespace PrimaryNamespace {
-    // Function in PrimaryNamespace
-    void functionInPrimaryNamespace() {}
+// Function in PrimaryNamespace
+void functionInPrimaryNamespace() {}
 // MD-PRIMARY-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-PRIMARY-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
-    // Class in PrimaryNamespace
-    class ClassInPrimaryNamespace {};
+// Class in PrimaryNamespace
+class ClassInPrimaryNamespace {};
 // MD-PRIMARY-CLASS-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-PRIMARY-CLASS-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
@@ -91,15 +87,15 @@ namespace PrimaryNamespace {
 // HTML-PRIMARY-CLASS: <h1>class ClassInPrimaryNamespace</h1>
 // HTML-PRIMARY-CLASS: <p> Class in PrimaryNamespace</p>
 
-    // Nested namespace
-    namespace NestedNamespace {
-        // Function in NestedNamespace
-        void functionInNestedNamespace() {}
+// Nested namespace
+namespace NestedNamespace {
+// Function in NestedNamespace
+void functionInNestedNamespace() {}
 // MD-NESTED-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-NESTED-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
-        // Class in NestedNamespace
-        class ClassInNestedNamespace {};
+// Class in NestedNamespace
+class ClassInNestedNamespace {};
 // MD-NESTED-CLASS-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-NESTED-CLASS-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
@@ -108,7 +104,7 @@ namespace PrimaryNamespace {
 
 // HTML-NESTED-CLASS: <h1>class ClassInNestedNamespace</h1>
 // HTML-NESTED-CLASS: <p> Class in NestedNamespace</p>
-    }
+} // namespace NestedNamespace
 
 // MD-NESTED-INDEX: # namespace NestedNamespace
 // MD-NESTED-INDEX: Nested namespace
@@ -127,7 +123,7 @@ namespace PrimaryNamespace {
 // HTML-NESTED-INDEX: <h3 id="{{([0-9A-F]{40})}}">functionInNestedNamespace</h3>
 // HTML-NESTED-INDEX: <p>void functionInNestedNamespace()</p>
 // HTML-NESTED-INDEX: <p> Function in NestedNamespace</p>
-}
+} // namespace PrimaryNamespace
 
 // MD-PRIMARY-INDEX: # namespace PrimaryNamespace
 // MD-PRIMARY-INDEX:  Primary Namespace
@@ -151,16 +147,15 @@ namespace PrimaryNamespace {
 // HTML-PRIMARY-INDEX: <p>void functionInPrimaryNamespace()</p>
 // HTML-PRIMARY-INDEX: <p> Function in PrimaryNamespace</p>
 
-
 // AnotherNamespace
 namespace AnotherNamespace {
-    // Function in AnotherNamespace
-    void functionInAnotherNamespace() {}
+// Function in AnotherNamespace
+void functionInAnotherNamespace() {}
 // MD-ANOTHER-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-ANOTHER-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
-    // Class in AnotherNamespace
-    class ClassInAnotherNamespace {};
+// Class in AnotherNamespace
+class ClassInAnotherNamespace {};
 // MD-ANOTHER-CLASS-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp#[[@LINE-1]]*
 // HTML-ANOTHER-CLASS-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}namespace.cpp</p>
 
@@ -170,7 +165,7 @@ namespace AnotherNamespace {
 // HTML-ANOTHER-CLASS: <h1>class ClassInAnotherNamespace</h1>
 // HTML-ANOTHER-CLASS: <p> Class in AnotherNamespace</p>
 
-}
+} // namespace AnotherNamespace
 
 // MD-ANOTHER-INDEX: # namespace AnotherNamespace
 // MD-ANOTHER-INDEX: AnotherNamespace
@@ -275,14 +270,12 @@ namespace AnotherNamespace {
 // HTML-GLOBAL-INDEX: <li>AnotherNamespace</li>
 // HTML-GLOBAL-INDEX: <li>PrimaryNamespace</li>
 
-
 // MD-GLOBAL-INDEX: # Global Namespace
 // MD-GLOBAL-INDEX: ## Namespaces
 // MD-GLOBAL-INDEX: * [@nonymous_namespace](..{{[\/]}}@nonymous_namespace{{[\/]}}index.md)
 // MD-GLOBAL-INDEX: * [AnotherNamespace](..{{[\/]}}AnotherNamespace{{[\/]}}index.md)
 // MD-GLOBAL-INDEX: * [PrimaryNamespace](..{{[\/]}}PrimaryNamespace{{[\/]}}index.md)
 
-
 // MD-ALL-FILES: # All Files
 // MD-ALL-FILES: ## [@nonymous_namespace](@nonymous_namespace{{[\/]}}index.md)
 // MD-ALL-FILES: ## [AnotherNamespace](AnotherNamespace{{[\/]}}index.md)
diff --git a/clang-tools-extra/test/clang-doc/single-file-public.cpp b/clang-tools-extra/test/clang-doc/single-file-public.cpp
index 060db05c6d992..a2a76b60c02db 100644
--- a/clang-tools-extra/test/clang-doc/single-file-public.cpp
+++ b/clang-tools-extra/test/clang-doc/single-file-public.cpp
@@ -9,10 +9,10 @@
 
 class Record {
 private:
-	void function_private();
+  void function_private();
 
 public:
-	void function_public();
+  void function_public();
 };
 
 void Record::function_private() {}
diff --git a/clang-tools-extra/test/clang-doc/templates.cpp b/clang-tools-extra/test/clang-doc/templates.cpp
index cab5426b7cefc..426a0b16befd4 100644
--- a/clang-tools-extra/test/clang-doc/templates.cpp
+++ b/clang-tools-extra/test/clang-doc/templates.cpp
@@ -19,7 +19,7 @@
 // MD: # Global Namespace
 // MD: ## Functions
 
-template<class... T>
+template <class... T>
 void ParamPackFunction(T... args);
 
 // YAML-NEXT: ChildFunctions:
@@ -44,7 +44,7 @@ void ParamPackFunction(T... args);
 // MD: ### ParamPackFunction
 // MD: *void ParamPackFunction(T... args)*
 
-template<typename T, int U = 1>
+template <typename T, int U = 1>
 void function(T x) {}
 
 // YAML-NEXT:   - USR:             '{{([0-9A-F]{40})}}'
@@ -70,7 +70,7 @@ void function(T x) {}
 // MD: *void function(T x)*
 // MD: *Defined at {{.*}}templates.cpp#[[# @LINE - 23]]*
 
-template<>
+template <>
 void function<bool, 0>(bool x) {}
 
 // YAML-NEXT:   - USR:             '{{([0-9A-F]{40})}}'
@@ -101,13 +101,13 @@ void function<bool, 0>(bool x) {}
 /// A Tuple type
 ///
 /// Does Tuple things.
-template<typename ...Tys>
-struct tuple{};
+template <typename... Tys>
+struct tuple {};
 
 /// A function with a tuple parameter
 ///
 /// \param t The input to func_with_tuple_param
-tuple<int,int,bool> func_with_tuple_param(tuple<int,int,bool> t){ return t;}
+tuple<int, int, bool> func_with_tuple_param(tuple<int, int, bool> t) { return t; }
 
 // YAML-NEXT:   - USR:             '{{([0-9A-F]{40})}}'
 // YAML-NEXT:    Name:            'func_with_tuple_param'
@@ -154,4 +154,3 @@ tuple<int,int,bool> func_with_tuple_param(tuple<int,int,bool> t){ return t;}
 // MD: *Defined at {{.*}}templates.cpp#[[# @LINE - 44]]*
 // MD:  A function with a tuple parameter
 // MD: **t** The input to func_with_tuple_param
-
diff --git a/clang-tools-extra/test/clang-doc/test-path-abs.cpp b/clang-tools-extra/test/clang-doc/test-path-abs.cpp
index 292a2a3b5474d..5b21fba4f11b1 100644
--- a/clang-tools-extra/test/clang-doc/test-path-abs.cpp
+++ b/clang-tools-extra/test/clang-doc/test-path-abs.cpp
@@ -3,4 +3,3 @@
 // RUN: FileCheck %s -input-file=%t/index_json.js  -check-prefix=JSON-INDEX
 
 // JSON-INDEX: var RootPath = "{{.*}}test-path-abs.cpp.tmp";
-
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-anon-record-fields.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-anon-record-fields.cpp
index ad6525276ff8a..1b4d4e924a721 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-anon-record-fields.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-anon-record-fields.cpp
@@ -33,29 +33,23 @@
 // RUN:     readability-identifier-naming.LocalConstantPointerPrefix: 'lc_', \
 // RUN:   }}'
 
-// FIXME: make this test case pass.
-// Currently not working because the CXXRecordDecl for the global anonymous
-// union is *not* collected as a top-level declaration.
-// https://github.com/llvm/llvm-project/issues/130618
-#if 0
 static union {
   int global;
-// FIXME-CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'global'
-// FIXME-CHECK-FIXES: {{^}}  int g_global;{{$}}
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'global'
+// CHECK-FIXES: {{^}}  int g_global;{{$}}
 
   const int global_const;
-// FIXME-CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global constant 'global_const'
-// FIXME-CHECK-FIXES: {{^}}  const int GLOBAL_CONST;{{$}}
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global constant 'global_const'
+// CHECK-FIXES: {{^}}  const int GLOBAL_CONST;{{$}}
 
   int *global_ptr;
-// FIXME-CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'global_ptr'
-// FIXME-CHECK-FIXES: {{^}}  int *GlobalPtr_Ptr;{{$}}
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'global_ptr'
+// CHECK-FIXES: {{^}}  int *GlobalPtr_Ptr;{{$}}
 
   int *const global_const_ptr;
-// FIXME-CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global constant pointer 'global_const_ptr'
-// FIXME-CHECK-FIXES: {{^}}  int *const GLOBAL_CONST_PTR_Ptr;{{$}}
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global constant pointer 'global_const_ptr'
+// CHECK-FIXES: {{^}}  int *const GLOBAL_CONST_PTR_Ptr;{{$}}
 };
-#endif
 
 namespace ns {
 
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
index a7956b4599b4f..448ef9ddf166c 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
@@ -66,12 +66,19 @@ class A { A(int); };
 // CHECK4-NOT: warning:
 // CHECK4-QUIET-NOT: warning:
 
+// CHECK: Suppressed 3 warnings (3 in non-user code)
 // CHECK: Use -header-filter=.* to display errors from all non-system headers.
 // CHECK-QUIET-NOT: Suppressed
+// CHECK2: Suppressed 1 warnings (1 in non-user code)
+// CHECK2: Use -header-filter=.* {{.*}}
 // CHECK2-QUIET-NOT: Suppressed
+// CHECK3: Suppressed 2 warnings (2 in non-user code)
+// CHECK3: Use -header-filter=.* {{.*}}
 // CHECK3-QUIET-NOT: Suppressed
 // CHECK4-NOT: Suppressed {{.*}} warnings
+// CHECK4-NOT: Use -header-filter=.* {{.*}}
 // CHECK4-QUIET-NOT: Suppressed
+// CHECK6: Suppressed 2 warnings (2 in non-user code)
 // CHECK6: Use -header-filter=.* {{.*}}
 
 int x = 123;
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp
index a25480e9aa39c..9fa990b6aac8c 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp
@@ -11,9 +11,9 @@
 // RUN: clang-tidy -help | FileCheck -check-prefix=CHECK-OPT-PRESENT %s
 
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -system-headers=true %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-SYSTEM-HEADERS %s
-// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -system-headers=false %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS --allow-empty %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -system-headers=false %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS %s
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -config='SystemHeaders: true' %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-SYSTEM-HEADERS %s
-// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -config='SystemHeaders: false' %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS --allow-empty %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -config='SystemHeaders: false' %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS %s
 
 #include <system_header.h>
 // CHECK-SYSTEM-HEADERS: system_header.h:1:13: warning: single-argument constructors must be marked explicit
diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index e881bf131d6c4..3dffa88e60193 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1561,6 +1561,9 @@ def __eq__(self, other):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def __hash__(self) -> int:
+        return self.hash
+
     def is_definition(self):
         """
         Returns true if the declaration pointed at by the cursor is also a
@@ -2035,6 +2038,13 @@ def lexical_parent(self):
 
         return self._lexical_parent
 
+    @property
+    def specialized_template(self) -> Cursor | None:
+        """Return the primary template that this cursor is a specialization of, if any."""
+        return Cursor.from_cursor_result(
+            conf.lib.clang_getSpecializedCursorTemplate(self), self
+        )
+
     @property
     def translation_unit(self):
         """Returns the TranslationUnit to which this Cursor belongs."""
@@ -2178,6 +2188,12 @@ def get_bitfield_width(self):
         """
         return conf.lib.clang_getFieldDeclBitWidth(self)  # type: ignore [no-any-return]
 
+    def has_attrs(self) -> bool:
+        """
+        Determine whether the given cursor has any attributes.
+        """
+        return bool(conf.lib.clang_Cursor_hasAttrs(self))
+
     @staticmethod
     def from_result(res, arg):
         assert isinstance(res, Cursor)
@@ -3932,6 +3948,7 @@ def set_property(self, property, value):
     ("clang_getCursorType", [Cursor], Type),
     ("clang_getCursorUSR", [Cursor], _CXString),
     ("clang_Cursor_getMangling", [Cursor], _CXString),
+    ("clang_Cursor_hasAttrs", [Cursor], c_uint),
     # ("clang_getCXTUResourceUsage",
     #  [TranslationUnit],
     #  CXTUResourceUsage),
diff --git a/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py
index beecfadd00858..5f38b1fe0d825 100644
--- a/clang/bindings/python/tests/cindex/test_cursor.py
+++ b/clang/bindings/python/tests/cindex/test_cursor.py
@@ -4,6 +4,7 @@
     AvailabilityKind,
     BinaryOperator,
     Config,
+    Cursor,
     CursorKind,
     PrintingPolicy,
     PrintingPolicyProperty,
@@ -995,3 +996,42 @@ def test_pretty_print(self):
         pp.set_property(PrintingPolicyProperty.Bool, False)
         self.assertEqual(pp.get_property(PrintingPolicyProperty.Bool), False)
         self.assertEqual(f.pretty_printed(pp), "void f(_Bool x) {\n}\n")
+
+    def test_hash(self):
+        def accumulate_cursors(cursor: Cursor, all_cursors: list):
+            all_cursors.append(cursor)
+            for child in cursor.get_children():
+                all_cursors = accumulate_cursors(child, all_cursors)
+            return all_cursors
+
+        tu = get_tu(children_test)
+        all_cursors = accumulate_cursors(tu.cursor, [])
+        cursor_hashes = set()
+        for cursor in all_cursors:
+            self.assertNotIn(hash(cursor), cursor_hashes)
+            cursor_hashes.add(hash(cursor))
+
+    def test_has_attrs(self):
+        tu = get_tu(
+            """
+struct A;
+struct A final {};
+
+struct B;
+struct B {};
+""",
+            lang="cpp",
+        )
+        A = get_cursor(tu, "A")
+        B = get_cursor(tu, "B")
+        self.assertTrue(A.get_definition().has_attrs())
+        self.assertFalse(B.get_definition().has_attrs())
+
+    def test_specialized_template(self):
+        tu = get_tu(template_arg_test, lang="cpp")
+        foos = get_cursors(tu, "foo")
+        prime_foo = foos[1].specialized_template
+
+        self.assertNotEqual(foos[0], foos[1])
+        self.assertEqual(foos[0], prime_foo)
+        self.assertIsNone(tu.cursor.specialized_template)
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 8182bccdd2da8..2f15c90ab1583 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -425,6 +425,11 @@ RISC-V Support
 
 - Add support for `-mtune=generic-ooo` (a generic out-of-order model).
 
+- Adds support for `__attribute__((interrupt("qci-nest")))` and
+  `__attribute__((interrupt("qci-nonest")))`. These use instructions from
+  Qualcomm's `Xqciint` extension to save and restore some GPRs in interrupt
+  service routines.
+
 CUDA/HIP Language Changes
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -458,11 +463,6 @@ AST Matchers
 - Ensure ``isDerivedFrom`` matches the correct base in case more than one alias exists.
 - Extend ``templateArgumentCountIs`` to support function and variable template
   specialization.
-- Move ``ast_matchers::MatchFinder::MatchFinderOptions`` to
-  ``ast_matchers::MatchFinderOptions``.
-- Add a boolean member ``SkipSystemHeaders`` to ``MatchFinderOptions``, and make
-  ``MatchASTConsumer`` receive a reference to ``MatchFinderOptions`` in the
-  constructor. This allows it to skip system headers when traversing the AST.
 
 clang-format
 ------------
@@ -525,6 +525,12 @@ Sanitizers
 
 Python Binding Changes
 ----------------------
+- Made ``Cursor`` hashable.
+- Added ``Cursor.has_attrs``, a binding for ``clang_Cursor_hasAttrs``, to check
+  whether a cursor has any attributes.
+- Added ``Cursor.specialized_template``, a binding for
+  ``clang_getSpecializedCursorTemplate``, to retrieve the primary template that
+  the cursor is a specialization of.
 - Added ``Type.get_methods``, a binding for ``clang_visitCXXMethods``, which
   allows visiting the methods of a class.
 
diff --git a/clang/include/clang/ASTMatchers/ASTMatchFinder.h b/clang/include/clang/ASTMatchers/ASTMatchFinder.h
index a6a8a350bfcd0..a387d9037b7da 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchFinder.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchFinder.h
@@ -50,24 +50,6 @@ namespace clang {
 
 namespace ast_matchers {
 
-/// A struct defining options for configuring the MatchFinder.
-struct MatchFinderOptions {
-  struct Profiling {
-    Profiling(llvm::StringMap<llvm::TimeRecord> &Records) : Records(Records) {}
-
-    /// Per bucket timing information.
-    llvm::StringMap<llvm::TimeRecord> &Records;
-  };
-
-  /// Enables per-check timers.
-  ///
-  /// It prints a report after match.
-  std::optional<Profiling> CheckProfiling;
-
-  /// Avoids matching declarations in system headers.
-  bool SkipSystemHeaders = false;
-};
-
 /// A class to allow finding matches over the Clang AST.
 ///
 /// After creation, you can add multiple matchers to the MatchFinder via
@@ -144,6 +126,21 @@ class MatchFinder {
     virtual void run() = 0;
   };
 
+  struct MatchFinderOptions {
+    struct Profiling {
+      Profiling(llvm::StringMap<llvm::TimeRecord> &Records)
+          : Records(Records) {}
+
+      /// Per bucket timing information.
+      llvm::StringMap<llvm::TimeRecord> &Records;
+    };
+
+    /// Enables per-check timers.
+    ///
+    /// It prints a report after match.
+    std::optional<Profiling> CheckProfiling;
+  };
+
   MatchFinder(MatchFinderOptions Options = MatchFinderOptions());
   ~MatchFinder();
 
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 1e9db55fbe301..0999d8065e9f5 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4395,8 +4395,8 @@ def HLSLControlFlowHint: StmtAttr {
   /// [branch]
   /// [flatten]
   let Spellings = [Microsoft<"branch">, Microsoft<"flatten">];
-  let Subjects = SubjectList<[IfStmt],
-                              ErrorDiag, "'if' statements">;
+  let Subjects = SubjectList<[IfStmt, SwitchStmt],
+                              ErrorDiag, "'if' and 'switch' statements">;
   let LangOpts = [HLSL];
   let Documentation = [InternalOnly];
 }
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 058fecd4e91ef..df24cca49aaae 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -92,10 +92,6 @@ def err_drv_hipspv_no_hip_path : Error<
   "'--hip-path' must be specified when offloading to SPIR-V unless '-nogpuinc' "
   "is given">;
 
-// TODO: Remove when COV6 is fully supported by ROCm.
-def warn_drv_amdgpu_cov6: Warning<
-  "code object v6 is still in development and not ready for production use yet;"
-  " use at your own risk">;
 def err_drv_undetermined_gpu_arch : Error<
   "cannot determine %0 architecture: %1; consider passing it via '%2'; "
   "environment variable CLANG_TOOLCHAIN_PROGRAM_TIMEOUT specifies the tool "
diff --git a/clang/include/clang/Sema/HLSLExternalSemaSource.h b/clang/include/clang/Sema/HLSLExternalSemaSource.h
index 3c7495e66055d..d93fb8c8eef6b 100644
--- a/clang/include/clang/Sema/HLSLExternalSemaSource.h
+++ b/clang/include/clang/Sema/HLSLExternalSemaSource.h
@@ -12,9 +12,8 @@
 #ifndef CLANG_SEMA_HLSLEXTERNALSEMASOURCE_H
 #define CLANG_SEMA_HLSLEXTERNALSEMASOURCE_H
 
-#include "llvm/ADT/DenseMap.h"
-
 #include "clang/Sema/ExternalSemaSource.h"
+#include "llvm/ADT/DenseMap.h"
 
 namespace clang {
 class NamespaceDecl;
@@ -27,14 +26,8 @@ class HLSLExternalSemaSource : public ExternalSemaSource {
   using CompletionFunction = std::function<void(CXXRecordDecl *)>;
   llvm::DenseMap<CXXRecordDecl *, CompletionFunction> Completions;
 
-  void defineHLSLVectorAlias();
-  void defineTrivialHLSLTypes();
-  void defineHLSLTypesWithForwardDeclarations();
-
-  void onCompletion(CXXRecordDecl *Record, CompletionFunction Fn);
-
 public:
-  ~HLSLExternalSemaSource() override;
+  ~HLSLExternalSemaSource() override {}
 
   /// Initialize the semantic source with the Sema instance
   /// being used to perform semantic analysis on the abstract syntax
@@ -47,6 +40,12 @@ class HLSLExternalSemaSource : public ExternalSemaSource {
   using ExternalASTSource::CompleteType;
   /// Complete an incomplete HLSL builtin type
   void CompleteType(TagDecl *Tag) override;
+
+private:
+  void defineTrivialHLSLTypes();
+  void defineHLSLVectorAlias();
+  void defineHLSLTypesWithForwardDeclarations();
+  void onCompletion(CXXRecordDecl *Record, CompletionFunction Fn);
 };
 
 } // namespace clang
diff --git a/clang/lib/ASTMatchers/ASTMatchFinder.cpp b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
index e347d0c54d9b0..e9ec7eff1e0ab 100644
--- a/clang/lib/ASTMatchers/ASTMatchFinder.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
@@ -28,7 +28,6 @@
 #include <deque>
 #include <memory>
 #include <set>
-#include <vector>
 
 namespace clang {
 namespace ast_matchers {
@@ -423,7 +422,7 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
                         public ASTMatchFinder {
 public:
   MatchASTVisitor(const MatchFinder::MatchersByType *Matchers,
-                  const MatchFinderOptions &Options)
+                  const MatchFinder::MatchFinderOptions &Options)
       : Matchers(Matchers), Options(Options), ActiveASTContext(nullptr) {}
 
   ~MatchASTVisitor() override {
@@ -1351,7 +1350,7 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
   /// We precalculate a list of matchers that pass the toplevel restrict check.
   llvm::DenseMap<ASTNodeKind, std::vector<unsigned short>> MatcherFiltersMap;
 
-  const MatchFinderOptions &Options;
+  const MatchFinder::MatchFinderOptions &Options;
   ASTContext *ActiveASTContext;
 
   // Maps a canonical type to its TypedefDecls.
@@ -1574,41 +1573,19 @@ bool MatchASTVisitor::TraverseAttr(Attr *AttrNode) {
 class MatchASTConsumer : public ASTConsumer {
 public:
   MatchASTConsumer(MatchFinder *Finder,
-                   MatchFinder::ParsingDoneTestCallback *ParsingDone,
-                   const MatchFinderOptions &Options)
-      : Finder(Finder), ParsingDone(ParsingDone), Options(Options) {}
+                   MatchFinder::ParsingDoneTestCallback *ParsingDone)
+      : Finder(Finder), ParsingDone(ParsingDone) {}
 
 private:
-  bool HandleTopLevelDecl(DeclGroupRef DG) override {
-    if (Options.SkipSystemHeaders) {
-      for (Decl *D : DG) {
-        if (!isInSystemHeader(D))
-          TraversalScope.push_back(D);
-      }
-    }
-    return true;
-  }
-
   void HandleTranslationUnit(ASTContext &Context) override {
-    if (!TraversalScope.empty())
-      Context.setTraversalScope(TraversalScope);
-
     if (ParsingDone != nullptr) {
       ParsingDone->run();
     }
     Finder->matchAST(Context);
   }
 
-  bool isInSystemHeader(Decl *D) {
-    const SourceManager &SM = D->getASTContext().getSourceManager();
-    const SourceLocation Loc = SM.getExpansionLoc(D->getBeginLoc());
-    return SM.isInSystemHeader(Loc);
-  }
-
   MatchFinder *Finder;
   MatchFinder::ParsingDoneTestCallback *ParsingDone;
-  const MatchFinderOptions &Options;
-  std::vector<Decl *> TraversalScope;
 };
 
 } // end namespace
@@ -1727,8 +1704,7 @@ bool MatchFinder::addDynamicMatcher(const internal::DynTypedMatcher &NodeMatch,
 }
 
 std::unique_ptr<ASTConsumer> MatchFinder::newASTConsumer() {
-  return std::make_unique<internal::MatchASTConsumer>(this, ParsingDone,
-                                                      Options);
+  return std::make_unique<internal::MatchASTConsumer>(this, ParsingDone);
 }
 
 void MatchFinder::match(const clang::DynTypedNode &Node, ASTContext &Context) {
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 9860f23dc9e28..3562b4ea22a24 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -2276,6 +2276,19 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
   // failure.
   llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default");
   SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock);
+  if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {
+    llvm::MDBuilder MDHelper(CGM.getLLVMContext());
+    llvm::ConstantInt *BranchHintConstant =
+        HLSLControlFlowAttr ==
+                HLSLControlFlowHintAttr::Spelling::Microsoft_branch
+            ? llvm::ConstantInt::get(CGM.Int32Ty, 1)
+            : llvm::ConstantInt::get(CGM.Int32Ty, 2);
+    llvm::Metadata *Vals[] = {MDHelper.createString("hlsl.controlflow.hint"),
+                              MDHelper.createConstant(BranchHintConstant)};
+    SwitchInsn->setMetadata("hlsl.controlflow.hint",
+                            llvm::MDNode::get(CGM.getLLVMContext(), Vals));
+  }
+
   if (PGO.haveRegionCounts()) {
     // Walk the SwitchCase list to find how many there are.
     uint64_t DefaultCount = 0;
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 157b9ff971add..7f04bcab9cd80 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -2748,12 +2748,6 @@ void tools::checkAMDGPUCodeObjectVersion(const Driver &D,
       if (Remnant || CodeObjVer < MinCodeObjVer || CodeObjVer > MaxCodeObjVer)
         D.Diag(diag::err_drv_invalid_int_value)
             << CodeObjArg->getAsString(Args) << CodeObjArg->getValue();
-
-      // COV6 is only supported by LLVM at the time of writing this, and it's
-      // expected to take some time before all ROCm components fully
-      // support it. In the meantime, make sure users are aware of this.
-      if (CodeObjVer == 6)
-        D.Diag(diag::warn_drv_amdgpu_cov6);
     }
   }
 }
diff --git a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
index 585e905c7bf5d..b0ff9075377d8 100644
--- a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
@@ -134,16 +134,18 @@ double4 abs(double4);
 /// \brief Returns the arccosine of the input value, \a Val.
 /// \param Val The input value.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_acos)
 half acos(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_acos)
 half2 acos(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_acos)
 half3 acos(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_acos)
 half4 acos(half4);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_acos)
 float acos(float);
@@ -447,16 +449,18 @@ double4 asdouble(uint4, uint4);
 /// \brief Returns the arcsine of the input value, \a Val.
 /// \param Val The input value.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_asin)
 half asin(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_asin)
 half2 asin(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_asin)
 half3 asin(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_asin)
 half4 asin(half4);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_asin)
 float asin(float);
@@ -475,16 +479,18 @@ float4 asin(float4);
 /// \brief Returns the arctangent of the input value, \a Val.
 /// \param Val The input value.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan)
 half atan(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan)
 half2 atan(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan)
 half3 atan(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan)
 half4 atan(half4);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan)
 float atan(float);
@@ -505,16 +511,18 @@ float4 atan(float4);
 /// \param y The y-coordinate.
 /// \param x The x-coordinate.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
 half atan2(half y, half x);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
 half2 atan2(half2 y, half2 x);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
 half3 atan2(half3 y, half3 x);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
 half4 atan2(half4 y, half4 x);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_atan2)
 float atan2(float y, float x);
@@ -721,16 +729,18 @@ float4 cos(float4);
 /// \brief Returns the hyperbolic cosine of the input value, \a Val.
 /// \param Val The input value.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_cosh)
 half cosh(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_cosh)
 half2 cosh(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_cosh)
 half3 cosh(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_cosh)
 half4 cosh(half4);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_cosh)
 float cosh(float);
@@ -2124,16 +2134,18 @@ float4 sin(float4);
 /// \brief Returns the hyperbolic sine of the input value, \a Val.
 /// \param Val The input value.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_sinh)
 half sinh(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_sinh)
 half2 sinh(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_sinh)
 half3 sinh(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_sinh)
 half4 sinh(half4);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_sinh)
 float sinh(float);
@@ -2215,16 +2227,18 @@ float4 step(float4, float4);
 /// \brief Returns the tangent of the input value, \a Val.
 /// \param Val The input value.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
 half tan(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
 half2 tan(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
 half3 tan(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
 half4 tan(half4);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tan)
 float tan(float);
@@ -2243,16 +2257,18 @@ float4 tan(float4);
 /// \brief Returns the hyperbolic tangent of the input value, \a Val.
 /// \param Val The input value.
 
-#ifdef __HLSL_ENABLE_16_BIT
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tanh)
 half tanh(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tanh)
 half2 tanh(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tanh)
 half3 tanh(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tanh)
 half4 tanh(half4);
-#endif
 
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_tanh)
 float tanh(float);
diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index 1a351684d133e..d3fe80f659f69 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -20,6 +20,7 @@ add_clang_library(clangSema
   DeclSpec.cpp
   DelayedDiagnostic.cpp
   HeuristicResolver.cpp
+  HLSLBuiltinTypeDeclBuilder.cpp
   HLSLExternalSemaSource.cpp
   IdentifierResolver.cpp
   JumpDiagnostics.cpp
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
new file mode 100644
index 0000000000000..6b7cc86fefa8f
--- /dev/null
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
@@ -0,0 +1,781 @@
+//===--- HLSLBuiltinTypeDeclBuilder.cpp - HLSL Builtin Type Decl Builder --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper classes for creating HLSL builtin class types. Used by external HLSL
+// sema source.
+//
+//===----------------------------------------------------------------------===//
+
+#include "HLSLBuiltinTypeDeclBuilder.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/Type.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaHLSL.h"
+#include "llvm/ADT/SmallVector.h"
+
+using namespace llvm::hlsl;
+
+namespace clang {
+
+namespace hlsl {
+
+namespace {
+
+static FunctionDecl *lookupBuiltinFunction(Sema &S, StringRef Name) {
+  IdentifierInfo &II =
+      S.getASTContext().Idents.get(Name, tok::TokenKind::identifier);
+  DeclarationNameInfo NameInfo =
+      DeclarationNameInfo(DeclarationName(&II), SourceLocation());
+  LookupResult R(S, NameInfo, Sema::LookupOrdinaryName);
+  // AllowBuiltinCreation is false but LookupDirect will create
+  // the builtin when searching the global scope anyways...
+  S.LookupName(R, S.getCurScope());
+  // FIXME: If the builtin function was user-declared in global scope,
+  // this assert *will* fail. Should this call LookupBuiltin instead?
+  assert(R.isSingleResult() &&
+         "Since this is a builtin it should always resolve!");
+  return cast<FunctionDecl>(R.getFoundDecl());
+}
+} // namespace
+
+// Builder for template arguments of builtin types. Used internally
+// by BuiltinTypeDeclBuilder.
+struct TemplateParameterListBuilder {
+  BuiltinTypeDeclBuilder &Builder;
+  llvm::SmallVector<NamedDecl *> Params;
+
+  TemplateParameterListBuilder(BuiltinTypeDeclBuilder &RB) : Builder(RB) {}
+  ~TemplateParameterListBuilder();
+
+  TemplateParameterListBuilder &
+  addTypeParameter(StringRef Name, QualType DefaultValue = QualType());
+
+  ConceptSpecializationExpr *
+  constructConceptSpecializationExpr(Sema &S, ConceptDecl *CD);
+
+  BuiltinTypeDeclBuilder &finalizeTemplateArgs(ConceptDecl *CD = nullptr);
+};
+
+// Builder for methods of builtin types. Allows adding methods to builtin types
+// using the builder pattern like this:
+//
+//   BuiltinTypeMethodBuilder(RecordBuilder, "MethodName", ReturnType)
+//       .addParam("param_name", Type, InOutModifier)
+//       .callBuiltin("builtin_name", BuiltinParams...)
+//       .finalizeMethod();
+//
+// The builder needs to have all of the method parameters before it can create
+// a CXXMethodDecl. It collects them in addParam calls and when a first
+// method that builds the body is called or when access to 'this` is needed it
+// creates the CXXMethodDecl and ParmVarDecls instances. These can then be
+// referenced from the body building methods. Destructor or an explicit call to
+// finalizeMethod() will complete the method definition.
+//
+// The callBuiltin helper method accepts constants via `Expr *` or placeholder
+// value arguments to indicate which function arguments to forward to the
+// builtin.
+//
+// If the method that is being built has a non-void return type the
+// finalizeMethod will create a return statent with the value of the last
+// statement (unless the last statement is already a ReturnStmt).
+struct BuiltinTypeMethodBuilder {
+private:
+  struct MethodParam {
+    const IdentifierInfo &NameII;
+    QualType Ty;
+    HLSLParamModifierAttr::Spelling Modifier;
+    MethodParam(const IdentifierInfo &NameII, QualType Ty,
+                HLSLParamModifierAttr::Spelling Modifier)
+        : NameII(NameII), Ty(Ty), Modifier(Modifier) {}
+  };
+
+  BuiltinTypeDeclBuilder &DeclBuilder;
+  DeclarationNameInfo NameInfo;
+  QualType ReturnTy;
+  CXXMethodDecl *Method;
+  bool IsConst;
+  llvm::SmallVector<MethodParam> Params;
+  llvm::SmallVector<Stmt *> StmtsList;
+
+  // Argument placeholders, inspired by std::placeholder. These are the indices
+  // of arguments to forward to `callBuiltin` and other method builder methods.
+  // Additional special values are:
+  //   Handle   - refers to the resource handle.
+  //   LastStmt - refers to the last statement in the method body; referencing
+  //              LastStmt will remove the statement from the method body since
+  //              it will be linked from the new expression being constructed.
+  enum class PlaceHolder { _0, _1, _2, _3, Handle = 128, LastStmt };
+
+  Expr *convertPlaceholder(PlaceHolder PH);
+  Expr *convertPlaceholder(Expr *E) { return E; }
+
+public:
+  friend BuiltinTypeDeclBuilder;
+
+  BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB, DeclarationName &Name,
+                           QualType ReturnTy, bool IsConst = false)
+      : DeclBuilder(DB), NameInfo(DeclarationNameInfo(Name, SourceLocation())),
+        ReturnTy(ReturnTy), Method(nullptr), IsConst(IsConst) {}
+
+  BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB, StringRef Name,
+                           QualType ReturnTy, bool IsConst = false);
+  BuiltinTypeMethodBuilder(const BuiltinTypeMethodBuilder &Other) = delete;
+
+  ~BuiltinTypeMethodBuilder() { finalizeMethod(); }
+
+  BuiltinTypeMethodBuilder &
+  operator=(const BuiltinTypeMethodBuilder &Other) = delete;
+
+  BuiltinTypeMethodBuilder &addParam(StringRef Name, QualType Ty,
+                                     HLSLParamModifierAttr::Spelling Modifier =
+                                         HLSLParamModifierAttr::Keyword_in);
+  template <typename... Ts>
+  BuiltinTypeMethodBuilder &callBuiltin(StringRef BuiltinName,
+                                        QualType ReturnType, Ts... ArgSpecs);
+  template <typename TLHS, typename TRHS>
+  BuiltinTypeMethodBuilder &assign(TLHS LHS, TRHS RHS);
+  template <typename T> BuiltinTypeMethodBuilder &dereference(T Ptr);
+  BuiltinTypeDeclBuilder &finalizeMethod();
+  Expr *getResourceHandleExpr();
+
+private:
+  void createMethodDecl();
+};
+
+TemplateParameterListBuilder::~TemplateParameterListBuilder() {
+  finalizeTemplateArgs();
+}
+
+TemplateParameterListBuilder &
+TemplateParameterListBuilder::addTypeParameter(StringRef Name,
+                                               QualType DefaultValue) {
+  assert(!Builder.Record->isCompleteDefinition() &&
+         "record is already complete");
+  ASTContext &AST = Builder.SemaRef.getASTContext();
+  unsigned Position = static_cast<unsigned>(Params.size());
+  auto *Decl = TemplateTypeParmDecl::Create(
+      AST, Builder.Record->getDeclContext(), SourceLocation(), SourceLocation(),
+      /* TemplateDepth */ 0, Position,
+      &AST.Idents.get(Name, tok::TokenKind::identifier),
+      /* Typename */ true,
+      /* ParameterPack */ false,
+      /* HasTypeConstraint*/ false);
+  if (!DefaultValue.isNull())
+    Decl->setDefaultArgument(AST,
+                             Builder.SemaRef.getTrivialTemplateArgumentLoc(
+                                 DefaultValue, QualType(), SourceLocation()));
+
+  Params.emplace_back(Decl);
+  return *this;
+}
+
+// The concept specialization expression (CSE) constructed in
+// constructConceptSpecializationExpr is constructed so that it
+// matches the CSE that is constructed when parsing the below C++ code:
+//
+// template<typename T>
+// concept is_typed_resource_element_compatible =
+// __builtin_hlsl_typed_resource_element_compatible<T>
+//
+// template<typename element_type> requires
+// is_typed_resource_element_compatible<element_type>
+// struct RWBuffer {
+//     element_type Val;
+// };
+//
+// int fn() {
+//     RWBuffer<int> Buf;
+// }
+//
+// When dumping the AST and filtering for "RWBuffer", the resulting AST
+// structure is what we're trying to construct below, specifically the
+// CSE portion.
+ConceptSpecializationExpr *
+TemplateParameterListBuilder::constructConceptSpecializationExpr(
+    Sema &S, ConceptDecl *CD) {
+  ASTContext &Context = S.getASTContext();
+  SourceLocation Loc = Builder.Record->getBeginLoc();
+  DeclarationNameInfo DNI(CD->getDeclName(), Loc);
+  NestedNameSpecifierLoc NNSLoc;
+  DeclContext *DC = Builder.Record->getDeclContext();
+  TemplateArgumentListInfo TALI(Loc, Loc);
+
+  // Assume that the concept decl has just one template parameter
+  // This parameter should have been added when CD was constructed
+  // in getTypedBufferConceptDecl
+  assert(CD->getTemplateParameters()->size() == 1 &&
+         "unexpected concept decl parameter count");
+  TemplateTypeParmDecl *ConceptTTPD =
+      dyn_cast<TemplateTypeParmDecl>(CD->getTemplateParameters()->getParam(0));
+
+  // this TemplateTypeParmDecl is the template for the resource, and is
+  // used to construct a template argumentthat will be used
+  // to construct the ImplicitConceptSpecializationDecl
+  TemplateTypeParmDecl *T = TemplateTypeParmDecl::Create(
+      Context,                          // AST context
+      Builder.Record->getDeclContext(), // DeclContext
+      SourceLocation(), SourceLocation(),
+      /*D=*/0,                    // Depth in the template parameter list
+      /*P=*/0,                    // Position in the template parameter list
+      /*Id=*/nullptr,             // Identifier for 'T'
+      /*Typename=*/true,          // Indicates this is a 'typename' or 'class'
+      /*ParameterPack=*/false,    // Not a parameter pack
+      /*HasTypeConstraint=*/false // Has no type constraint
+  );
+
+  T->setDeclContext(DC);
+
+  QualType ConceptTType = Context.getTypeDeclType(ConceptTTPD);
+
+  // this is the 2nd template argument node, on which
+  // the concept constraint is actually being applied: 'element_type'
+  TemplateArgument ConceptTA = TemplateArgument(ConceptTType);
+
+  QualType CSETType = Context.getTypeDeclType(T);
+
+  // this is the 1st template argument node, which represents
+  // the abstract type that a concept would refer to: 'T'
+  TemplateArgument CSETA = TemplateArgument(CSETType);
+
+  ImplicitConceptSpecializationDecl *ImplicitCSEDecl =
+      ImplicitConceptSpecializationDecl::Create(
+          Context, Builder.Record->getDeclContext(), Loc, {CSETA});
+
+  // Constraint satisfaction is used to construct the
+  // ConceptSpecailizationExpr, and represents the 2nd Template Argument,
+  // located at the bottom of the sample AST above.
+  const ConstraintSatisfaction CS(CD, {ConceptTA});
+  TemplateArgumentLoc TAL =
+      S.getTrivialTemplateArgumentLoc(ConceptTA, QualType(), SourceLocation());
+
+  TALI.addArgument(TAL);
+  const ASTTemplateArgumentListInfo *ATALI =
+      ASTTemplateArgumentListInfo::Create(Context, TALI);
+
+  // In the concept reference, ATALI is what adds the extra
+  // TemplateArgument node underneath CSE
+  ConceptReference *CR =
+      ConceptReference::Create(Context, NNSLoc, Loc, DNI, CD, CD, ATALI);
+
+  ConceptSpecializationExpr *CSE =
+      ConceptSpecializationExpr::Create(Context, CR, ImplicitCSEDecl, &CS);
+
+  return CSE;
+}
+
+BuiltinTypeDeclBuilder &
+TemplateParameterListBuilder::finalizeTemplateArgs(ConceptDecl *CD) {
+  if (Params.empty())
+    return Builder;
+
+  ASTContext &AST = Builder.SemaRef.Context;
+  ConceptSpecializationExpr *CSE =
+      CD ? constructConceptSpecializationExpr(Builder.SemaRef, CD) : nullptr;
+  auto *ParamList = TemplateParameterList::Create(
+      AST, SourceLocation(), SourceLocation(), Params, SourceLocation(), CSE);
+  Builder.Template = ClassTemplateDecl::Create(
+      AST, Builder.Record->getDeclContext(), SourceLocation(),
+      DeclarationName(Builder.Record->getIdentifier()), ParamList,
+      Builder.Record);
+
+  Builder.Record->setDescribedClassTemplate(Builder.Template);
+  Builder.Template->setImplicit(true);
+  Builder.Template->setLexicalDeclContext(Builder.Record->getDeclContext());
+
+  // NOTE: setPreviousDecl before addDecl so new decl replace old decl when
+  // make visible.
+  Builder.Template->setPreviousDecl(Builder.PrevTemplate);
+  Builder.Record->getDeclContext()->addDecl(Builder.Template);
+  Params.clear();
+
+  QualType T = Builder.Template->getInjectedClassNameSpecialization();
+  T = AST.getInjectedClassNameType(Builder.Record, T);
+
+  return Builder;
+}
+
+Expr *BuiltinTypeMethodBuilder::convertPlaceholder(PlaceHolder PH) {
+  if (PH == PlaceHolder::Handle)
+    return getResourceHandleExpr();
+
+  if (PH == PlaceHolder::LastStmt) {
+    assert(!StmtsList.empty() && "no statements in the list");
+    Stmt *LastStmt = StmtsList.pop_back_val();
+    assert(isa<ValueStmt>(LastStmt) && "last statement does not have a value");
+    return cast<ValueStmt>(LastStmt)->getExprStmt();
+  }
+
+  ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
+  ParmVarDecl *ParamDecl = Method->getParamDecl(static_cast<unsigned>(PH));
+  return DeclRefExpr::Create(
+      AST, NestedNameSpecifierLoc(), SourceLocation(), ParamDecl, false,
+      DeclarationNameInfo(ParamDecl->getDeclName(), SourceLocation()),
+      ParamDecl->getType(), VK_PRValue);
+}
+
+BuiltinTypeMethodBuilder::BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB,
+                                                   StringRef Name,
+                                                   QualType ReturnTy,
+                                                   bool IsConst)
+    : DeclBuilder(DB), ReturnTy(ReturnTy), Method(nullptr), IsConst(IsConst) {
+  const IdentifierInfo &II =
+      DB.SemaRef.getASTContext().Idents.get(Name, tok::TokenKind::identifier);
+  NameInfo = DeclarationNameInfo(DeclarationName(&II), SourceLocation());
+}
+
+BuiltinTypeMethodBuilder &
+BuiltinTypeMethodBuilder::addParam(StringRef Name, QualType Ty,
+                                   HLSLParamModifierAttr::Spelling Modifier) {
+  assert(Method == nullptr && "Cannot add param, method already created");
+  const IdentifierInfo &II = DeclBuilder.SemaRef.getASTContext().Idents.get(
+      Name, tok::TokenKind::identifier);
+  Params.emplace_back(II, Ty, Modifier);
+  return *this;
+}
+
+void BuiltinTypeMethodBuilder::createMethodDecl() {
+  assert(Method == nullptr && "Method already created");
+
+  // create method type
+  ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
+  SmallVector<QualType> ParamTypes;
+  for (MethodParam &MP : Params)
+    ParamTypes.emplace_back(MP.Ty);
+
+  FunctionProtoType::ExtProtoInfo ExtInfo;
+  if (IsConst)
+    ExtInfo.TypeQuals.addConst();
+
+  QualType MethodTy = AST.getFunctionType(ReturnTy, ParamTypes, ExtInfo);
+
+  // create method decl
+  auto *TSInfo = AST.getTrivialTypeSourceInfo(MethodTy, SourceLocation());
+  Method = CXXMethodDecl::Create(
+      AST, DeclBuilder.Record, SourceLocation(), NameInfo, MethodTy, TSInfo,
+      SC_None, false, false, ConstexprSpecKind::Unspecified, SourceLocation());
+
+  // create params & set them to the function prototype
+  SmallVector<ParmVarDecl *> ParmDecls;
+  auto FnProtoLoc =
+      Method->getTypeSourceInfo()->getTypeLoc().getAs<FunctionProtoTypeLoc>();
+  for (int I = 0, E = Params.size(); I != E; I++) {
+    MethodParam &MP = Params[I];
+    ParmVarDecl *Parm = ParmVarDecl::Create(
+        AST, Method->getDeclContext(), SourceLocation(), SourceLocation(),
+        &MP.NameII, MP.Ty,
+        AST.getTrivialTypeSourceInfo(MP.Ty, SourceLocation()), SC_None,
+        nullptr);
+    if (MP.Modifier != HLSLParamModifierAttr::Keyword_in) {
+      auto *Mod =
+          HLSLParamModifierAttr::Create(AST, SourceRange(), MP.Modifier);
+      Parm->addAttr(Mod);
+    }
+    ParmDecls.push_back(Parm);
+    FnProtoLoc.setParam(I, Parm);
+  }
+  Method->setParams({ParmDecls});
+}
+
+Expr *BuiltinTypeMethodBuilder::getResourceHandleExpr() {
+  // The first statement added to a method or access to 'this' creates the
+  // declaration.
+  if (!Method)
+    createMethodDecl();
+
+  ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
+  CXXThisExpr *This = CXXThisExpr::Create(
+      AST, SourceLocation(), Method->getFunctionObjectParameterType(), true);
+  FieldDecl *HandleField = DeclBuilder.getResourceHandleField();
+  return MemberExpr::CreateImplicit(AST, This, false, HandleField,
+                                    HandleField->getType(), VK_LValue,
+                                    OK_Ordinary);
+}
+
+template <typename... Ts>
+BuiltinTypeMethodBuilder &
+BuiltinTypeMethodBuilder::callBuiltin(StringRef BuiltinName,
+                                      QualType ReturnType, Ts... ArgSpecs) {
+  std::array<Expr *, sizeof...(ArgSpecs)> Args{
+      convertPlaceholder(std::forward<Ts>(ArgSpecs))...};
+
+  // The first statement added to a method or access to 'this` creates the
+  // declaration.
+  if (!Method)
+    createMethodDecl();
+
+  ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
+  FunctionDecl *FD = lookupBuiltinFunction(DeclBuilder.SemaRef, BuiltinName);
+  DeclRefExpr *DRE = DeclRefExpr::Create(
+      AST, NestedNameSpecifierLoc(), SourceLocation(), FD, false,
+      FD->getNameInfo(), AST.BuiltinFnTy, VK_PRValue);
+
+  if (ReturnType.isNull())
+    ReturnType = FD->getReturnType();
+
+  Expr *Call = CallExpr::Create(AST, DRE, Args, ReturnType, VK_PRValue,
+                                SourceLocation(), FPOptionsOverride());
+  StmtsList.push_back(Call);
+  return *this;
+}
+
+template <typename TLHS, typename TRHS>
+BuiltinTypeMethodBuilder &BuiltinTypeMethodBuilder::assign(TLHS LHS, TRHS RHS) {
+  Expr *LHSExpr = convertPlaceholder(LHS);
+  Expr *RHSExpr = convertPlaceholder(RHS);
+  Stmt *AssignStmt = BinaryOperator::Create(
+      DeclBuilder.SemaRef.getASTContext(), LHSExpr, RHSExpr, BO_Assign,
+      LHSExpr->getType(), ExprValueKind::VK_PRValue,
+      ExprObjectKind::OK_Ordinary, SourceLocation(), FPOptionsOverride());
+  StmtsList.push_back(AssignStmt);
+  return *this;
+}
+
+template <typename T>
+BuiltinTypeMethodBuilder &BuiltinTypeMethodBuilder::dereference(T Ptr) {
+  Expr *PtrExpr = convertPlaceholder(Ptr);
+  Expr *Deref =
+      UnaryOperator::Create(DeclBuilder.SemaRef.getASTContext(), PtrExpr,
+                            UO_Deref, PtrExpr->getType()->getPointeeType(),
+                            VK_PRValue, OK_Ordinary, SourceLocation(),
+                            /*CanOverflow=*/false, FPOptionsOverride());
+  StmtsList.push_back(Deref);
+  return *this;
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeMethodBuilder::finalizeMethod() {
+  assert(!DeclBuilder.Record->isCompleteDefinition() &&
+         "record is already complete");
+  assert(Method != nullptr &&
+         "method decl not created; are you missing a call to build the body?");
+
+  if (!Method->hasBody()) {
+    ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
+    assert((ReturnTy == AST.VoidTy || !StmtsList.empty()) &&
+           "nothing to return from non-void method");
+    if (ReturnTy != AST.VoidTy) {
+      if (Expr *LastExpr = dyn_cast<Expr>(StmtsList.back())) {
+        assert(AST.hasSameUnqualifiedType(LastExpr->getType(),
+                                          ReturnTy.getNonReferenceType()) &&
+               "Return type of the last statement must match the return type "
+               "of the method");
+        if (!isa<ReturnStmt>(LastExpr)) {
+          StmtsList.pop_back();
+          StmtsList.push_back(
+              ReturnStmt::Create(AST, SourceLocation(), LastExpr, nullptr));
+        }
+      }
+    }
+
+    Method->setBody(CompoundStmt::Create(AST, StmtsList, FPOptionsOverride(),
+                                         SourceLocation(), SourceLocation()));
+    Method->setLexicalDeclContext(DeclBuilder.Record);
+    Method->setAccess(AccessSpecifier::AS_public);
+    Method->addAttr(AlwaysInlineAttr::CreateImplicit(
+        AST, SourceRange(), AlwaysInlineAttr::CXX11_clang_always_inline));
+    DeclBuilder.Record->addDecl(Method);
+  }
+  return DeclBuilder;
+}
+
+BuiltinTypeDeclBuilder::BuiltinTypeDeclBuilder(Sema &SemaRef, CXXRecordDecl *R)
+    : SemaRef(SemaRef), Record(R) {
+  Record->startDefinition();
+  Template = Record->getDescribedClassTemplate();
+}
+
+BuiltinTypeDeclBuilder::BuiltinTypeDeclBuilder(Sema &SemaRef,
+                                               NamespaceDecl *Namespace,
+                                               StringRef Name)
+    : SemaRef(SemaRef), HLSLNamespace(Namespace) {
+  ASTContext &AST = SemaRef.getASTContext();
+  IdentifierInfo &II = AST.Idents.get(Name, tok::TokenKind::identifier);
+
+  LookupResult Result(SemaRef, &II, SourceLocation(), Sema::LookupTagName);
+  CXXRecordDecl *PrevDecl = nullptr;
+  if (SemaRef.LookupQualifiedName(Result, HLSLNamespace)) {
+    // Declaration already exists (from precompiled headers)
+    NamedDecl *Found = Result.getFoundDecl();
+    if (auto *TD = dyn_cast<ClassTemplateDecl>(Found)) {
+      PrevDecl = TD->getTemplatedDecl();
+      PrevTemplate = TD;
+    } else
+      PrevDecl = dyn_cast<CXXRecordDecl>(Found);
+    assert(PrevDecl && "Unexpected lookup result type.");
+  }
+
+  if (PrevDecl && PrevDecl->isCompleteDefinition()) {
+    Record = PrevDecl;
+    Template = PrevTemplate;
+    return;
+  }
+
+  Record = CXXRecordDecl::Create(AST, TagDecl::TagKind::Class, HLSLNamespace,
+                                 SourceLocation(), SourceLocation(), &II,
+                                 PrevDecl, true);
+  Record->setImplicit(true);
+  Record->setLexicalDeclContext(HLSLNamespace);
+  Record->setHasExternalLexicalStorage();
+
+  // Don't let anyone derive from built-in types.
+  Record->addAttr(
+      FinalAttr::CreateImplicit(AST, SourceRange(), FinalAttr::Keyword_final));
+}
+
+BuiltinTypeDeclBuilder::~BuiltinTypeDeclBuilder() {
+  if (HLSLNamespace && !Template && Record->getDeclContext() == HLSLNamespace)
+    HLSLNamespace->addDecl(Record);
+}
+
+CXXRecordDecl *BuiltinTypeDeclBuilder::finalizeForwardDeclaration() {
+  // Force the QualType to be generated for the record declaration. In most
+  // cases this will happen naturally when something uses the type the
+  // QualType gets lazily created. Unfortunately, with our injected types if a
+  // type isn't used in a translation unit the QualType may not get
+  // automatically generated before a PCH is generated. To resolve this we
+  // just force that the QualType is generated after we create a forward
+  // declaration.
+  (void)Record->getASTContext().getRecordType(Record);
+  return Record;
+}
+
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addMemberVariable(StringRef Name, QualType Type,
+                                          llvm::ArrayRef<Attr *> Attrs,
+                                          AccessSpecifier Access) {
+  assert(!Record->isCompleteDefinition() && "record is already complete");
+  assert(Record->isBeingDefined() &&
+         "Definition must be started before adding members!");
+  ASTContext &AST = Record->getASTContext();
+
+  IdentifierInfo &II = AST.Idents.get(Name, tok::TokenKind::identifier);
+  TypeSourceInfo *MemTySource =
+      AST.getTrivialTypeSourceInfo(Type, SourceLocation());
+  auto *Field = FieldDecl::Create(
+      AST, Record, SourceLocation(), SourceLocation(), &II, Type, MemTySource,
+      nullptr, false, InClassInitStyle::ICIS_NoInit);
+  Field->setAccess(Access);
+  Field->setImplicit(true);
+  for (Attr *A : Attrs) {
+    if (A)
+      Field->addAttr(A);
+  }
+
+  Record->addDecl(Field);
+  Fields[Name] = Field;
+  return *this;
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addHandleMember(
+    ResourceClass RC, bool IsROV, bool RawBuffer, AccessSpecifier Access) {
+  assert(!Record->isCompleteDefinition() && "record is already complete");
+
+  ASTContext &Ctx = SemaRef.getASTContext();
+  TypeSourceInfo *ElementTypeInfo =
+      Ctx.getTrivialTypeSourceInfo(getHandleElementType(), SourceLocation());
+
+  // add handle member with resource type attributes
+  QualType AttributedResTy = QualType();
+  SmallVector<const Attr *> Attrs = {
+      HLSLResourceClassAttr::CreateImplicit(Ctx, RC),
+      IsROV ? HLSLROVAttr::CreateImplicit(Ctx) : nullptr,
+      RawBuffer ? HLSLRawBufferAttr::CreateImplicit(Ctx) : nullptr,
+      ElementTypeInfo
+          ? HLSLContainedTypeAttr::CreateImplicit(Ctx, ElementTypeInfo)
+          : nullptr};
+  if (CreateHLSLAttributedResourceType(SemaRef, Ctx.HLSLResourceTy, Attrs,
+                                       AttributedResTy))
+    addMemberVariable("__handle", AttributedResTy, {}, Access);
+  return *this;
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addDefaultHandleConstructor() {
+  if (Record->isCompleteDefinition())
+    return *this;
+  ASTContext &AST = Record->getASTContext();
+
+  QualType ConstructorType =
+      AST.getFunctionType(AST.VoidTy, {}, FunctionProtoType::ExtProtoInfo());
+
+  CanQualType CanTy = Record->getTypeForDecl()->getCanonicalTypeUnqualified();
+  DeclarationName Name = AST.DeclarationNames.getCXXConstructorName(CanTy);
+  CXXConstructorDecl *Constructor = CXXConstructorDecl::Create(
+      AST, Record, SourceLocation(),
+      DeclarationNameInfo(Name, SourceLocation()), ConstructorType,
+      AST.getTrivialTypeSourceInfo(ConstructorType, SourceLocation()),
+      ExplicitSpecifier(), false, true, false, ConstexprSpecKind::Unspecified);
+
+  Constructor->setBody(CompoundStmt::Create(
+      AST, {}, FPOptionsOverride(), SourceLocation(), SourceLocation()));
+  Constructor->setAccess(AccessSpecifier::AS_public);
+  Record->addDecl(Constructor);
+  return *this;
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addArraySubscriptOperators() {
+  ASTContext &AST = Record->getASTContext();
+  DeclarationName Subscript =
+      AST.DeclarationNames.getCXXOperatorName(OO_Subscript);
+
+  addHandleAccessFunction(Subscript, /*IsConst=*/true, /*IsRef=*/true);
+  addHandleAccessFunction(Subscript, /*IsConst=*/false, /*IsRef=*/true);
+  return *this;
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addLoadMethods() {
+  if (Record->isCompleteDefinition())
+    return *this;
+
+  ASTContext &AST = Record->getASTContext();
+  IdentifierInfo &II = AST.Idents.get("Load", tok::TokenKind::identifier);
+  DeclarationName Load(&II);
+  // TODO: We also need versions with status for CheckAccessFullyMapped.
+  addHandleAccessFunction(Load, /*IsConst=*/false, /*IsRef=*/false);
+
+  return *this;
+}
+
+FieldDecl *BuiltinTypeDeclBuilder::getResourceHandleField() {
+  auto I = Fields.find("__handle");
+  assert(I != Fields.end() &&
+         I->second->getType()->isHLSLAttributedResourceType() &&
+         "record does not have resource handle field");
+  return I->second;
+}
+
+QualType BuiltinTypeDeclBuilder::getFirstTemplateTypeParam() {
+  assert(Template && "record it not a template");
+  if (const auto *TTD = dyn_cast<TemplateTypeParmDecl>(
+          Template->getTemplateParameters()->getParam(0))) {
+    return QualType(TTD->getTypeForDecl(), 0);
+  }
+  return QualType();
+}
+
+QualType BuiltinTypeDeclBuilder::getHandleElementType() {
+  if (Template)
+    return getFirstTemplateTypeParam();
+  // TODO: Should we default to VoidTy? Using `i8` is arguably ambiguous.
+  return SemaRef.getASTContext().Char8Ty;
+}
+
+// BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::startDefinition() {
+//   assert(!Record->isCompleteDefinition() && "record is already complete");
+//   Record->startDefinition();
+//   return *this;
+// }
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::completeDefinition() {
+  assert(!Record->isCompleteDefinition() && "record is already complete");
+  assert(Record->isBeingDefined() &&
+         "Definition must be started before completing it.");
+
+  Record->completeDefinition();
+  return *this;
+}
+
+Expr *BuiltinTypeDeclBuilder::getConstantIntExpr(int value) {
+  ASTContext &AST = SemaRef.getASTContext();
+  return IntegerLiteral::Create(
+      AST, llvm::APInt(AST.getTypeSize(AST.IntTy), value, true), AST.IntTy,
+      SourceLocation());
+}
+
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addSimpleTemplateParams(ArrayRef<StringRef> Names,
+                                                ConceptDecl *CD = nullptr) {
+  if (Record->isCompleteDefinition()) {
+    assert(Template && "existing record it not a template");
+    assert(Template->getTemplateParameters()->size() == Names.size() &&
+           "template param count mismatch");
+    return *this;
+  }
+
+  TemplateParameterListBuilder Builder = TemplateParameterListBuilder(*this);
+  for (StringRef Name : Names)
+    Builder.addTypeParameter(Name);
+  return Builder.finalizeTemplateArgs(CD);
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addIncrementCounterMethod() {
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+  return BuiltinTypeMethodBuilder(*this, "IncrementCounter",
+                                  SemaRef.getASTContext().UnsignedIntTy)
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", QualType(),
+                   PH::Handle, getConstantIntExpr(1))
+      .finalizeMethod();
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addDecrementCounterMethod() {
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+  return BuiltinTypeMethodBuilder(*this, "DecrementCounter",
+                                  SemaRef.getASTContext().UnsignedIntTy)
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", QualType(),
+                   PH::Handle, getConstantIntExpr(-1))
+      .finalizeMethod();
+}
+
+BuiltinTypeDeclBuilder &
+BuiltinTypeDeclBuilder::addHandleAccessFunction(DeclarationName &Name,
+                                                bool IsConst, bool IsRef) {
+  assert(!Record->isCompleteDefinition() && "record is already complete");
+  ASTContext &AST = SemaRef.getASTContext();
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+
+  QualType ElemTy = getHandleElementType();
+  // TODO: Map to an hlsl_device address space.
+  QualType ElemPtrTy = AST.getPointerType(ElemTy);
+  QualType ReturnTy = ElemTy;
+  if (IsConst)
+    ReturnTy.addConst();
+  if (IsRef)
+    ReturnTy = AST.getLValueReferenceType(ReturnTy);
+
+  return BuiltinTypeMethodBuilder(*this, Name, ReturnTy, IsConst)
+      .addParam("Index", AST.UnsignedIntTy)
+      .callBuiltin("__builtin_hlsl_resource_getpointer", ElemPtrTy, PH::Handle,
+                   PH::_0)
+      .dereference(PH::LastStmt)
+      .finalizeMethod();
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() {
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+  ASTContext &AST = SemaRef.getASTContext();
+  QualType ElemTy = getHandleElementType();
+  return BuiltinTypeMethodBuilder(*this, "Append", AST.VoidTy)
+      .addParam("value", ElemTy)
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", AST.UnsignedIntTy,
+                   PH::Handle, getConstantIntExpr(1))
+      .callBuiltin("__builtin_hlsl_resource_getpointer",
+                   AST.getPointerType(ElemTy), PH::Handle, PH::LastStmt)
+      .dereference(PH::LastStmt)
+      .assign(PH::LastStmt, PH::_0)
+      .finalizeMethod();
+}
+
+BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addConsumeMethod() {
+  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
+  ASTContext &AST = SemaRef.getASTContext();
+  QualType ElemTy = getHandleElementType();
+  return BuiltinTypeMethodBuilder(*this, "Consume", ElemTy)
+      .callBuiltin("__builtin_hlsl_buffer_update_counter", AST.UnsignedIntTy,
+                   PH::Handle, getConstantIntExpr(-1))
+      .callBuiltin("__builtin_hlsl_resource_getpointer",
+                   AST.getPointerType(ElemTy), PH::Handle, PH::LastStmt)
+      .dereference(PH::LastStmt)
+      .finalizeMethod();
+}
+
+} // namespace hlsl
+} // namespace clang
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
new file mode 100644
index 0000000000000..dbf54dfd9ecd9
--- /dev/null
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
@@ -0,0 +1,102 @@
+//===--- HLSLBuiltinTypeDeclBuilder.h - HLSL Builtin Type Decl Builder  ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper classes for creating HLSL builtin class types. Used by external HLSL
+// sema source.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_HLSLBUILTINTYPEDECLBUILDER_H
+#define LLVM_CLANG_SEMA_HLSLBUILTINTYPEDECLBUILDER_H
+
+#include "clang/AST/Type.h"
+#include "clang/Sema/Sema.h"
+#include "llvm/ADT/StringMap.h"
+
+using llvm::hlsl::ResourceClass;
+
+namespace clang {
+
+class ClassTemplateDecl;
+class NamespaceDecl;
+class CXXRecordDecl;
+class FieldDecl;
+
+namespace hlsl {
+
+// Builder for builtin HLSL class types such as HLSL resource classes.
+// Allows creating declaration of builtin types using the builder pattern
+// like this:
+//
+//   Decl = BuiltinTypeDeclBuilder(Sema, Namespace, "BuiltinClassName")
+//           .addSimpleTemplateParams({"T"}, Concept)
+//           .finalizeForwardDeclaration();
+//
+// And then completing the type like this:
+//
+//   BuiltinTypeDeclBuilder(Sema, Decl)
+//          .addDefaultHandleConstructor();
+//          .addLoadMethods()
+//          .completeDefinition();
+//
+class BuiltinTypeDeclBuilder {
+private:
+  Sema &SemaRef;
+  CXXRecordDecl *Record = nullptr;
+  ClassTemplateDecl *Template = nullptr;
+  ClassTemplateDecl *PrevTemplate = nullptr;
+  NamespaceDecl *HLSLNamespace = nullptr;
+  llvm::StringMap<FieldDecl *> Fields;
+
+public:
+  friend struct TemplateParameterListBuilder;
+  friend struct BuiltinTypeMethodBuilder;
+
+  BuiltinTypeDeclBuilder(Sema &SemaRef, CXXRecordDecl *R);
+  BuiltinTypeDeclBuilder(Sema &SemaRef, NamespaceDecl *Namespace,
+                         StringRef Name);
+  ~BuiltinTypeDeclBuilder();
+
+  BuiltinTypeDeclBuilder &addSimpleTemplateParams(ArrayRef<StringRef> Names,
+                                                  ConceptDecl *CD);
+  CXXRecordDecl *finalizeForwardDeclaration();
+  BuiltinTypeDeclBuilder &completeDefinition();
+
+  BuiltinTypeDeclBuilder &
+  addMemberVariable(StringRef Name, QualType Type, llvm::ArrayRef<Attr *> Attrs,
+                    AccessSpecifier Access = AccessSpecifier::AS_private);
+
+  BuiltinTypeDeclBuilder &
+  addHandleMember(ResourceClass RC, bool IsROV, bool RawBuffer,
+                  AccessSpecifier Access = AccessSpecifier::AS_private);
+  BuiltinTypeDeclBuilder &addArraySubscriptOperators();
+
+  // Builtin types methods
+  BuiltinTypeDeclBuilder &addDefaultHandleConstructor();
+
+  // Builtin types methods
+  BuiltinTypeDeclBuilder &addLoadMethods();
+  BuiltinTypeDeclBuilder &addIncrementCounterMethod();
+  BuiltinTypeDeclBuilder &addDecrementCounterMethod();
+  BuiltinTypeDeclBuilder &addHandleAccessFunction(DeclarationName &Name,
+                                                  bool IsConst, bool IsRef);
+  BuiltinTypeDeclBuilder &addAppendMethod();
+  BuiltinTypeDeclBuilder &addConsumeMethod();
+
+private:
+  FieldDecl *getResourceHandleField();
+  QualType getFirstTemplateTypeParam();
+  QualType getHandleElementType();
+  Expr *getConstantIntExpr(int value);
+};
+
+} // namespace hlsl
+
+} // namespace clang
+
+#endif // LLVM_CLANG_SEMA_HLSLBUILTINTYPEDECLBUILDER_H
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index cfa49029a2fb1..f5477ac912693 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -10,6 +10,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Sema/HLSLExternalSemaSource.h"
+#include "HLSLBuiltinTypeDeclBuilder.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
@@ -21,741 +22,11 @@
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Frontend/HLSL/HLSLResource.h"
-#include "llvm/Support/ErrorHandling.h"
-
-#include <functional>
 
 using namespace clang;
 using namespace llvm::hlsl;
 
-static FunctionDecl *lookupBuiltinFunction(Sema &S, StringRef Name);
-
-namespace {
-
-struct TemplateParameterListBuilder;
-
-class BuiltinTypeDeclBuilder {
-  ClassTemplateDecl *Template = nullptr;
-  ClassTemplateDecl *PrevTemplate = nullptr;
-  NamespaceDecl *HLSLNamespace = nullptr;
-  llvm::StringMap<FieldDecl *> Fields;
-
-public:
-  Sema &SemaRef;
-  CXXRecordDecl *Record = nullptr;
-  friend struct TemplateParameterListBuilder;
-
-  BuiltinTypeDeclBuilder(Sema &SemaRef, CXXRecordDecl *R)
-      : SemaRef(SemaRef), Record(R) {
-    Record->startDefinition();
-    Template = Record->getDescribedClassTemplate();
-  }
-
-  BuiltinTypeDeclBuilder(Sema &SemaRef, NamespaceDecl *Namespace,
-                         StringRef Name)
-      : HLSLNamespace(Namespace), SemaRef(SemaRef) {
-    ASTContext &AST = SemaRef.getASTContext();
-    IdentifierInfo &II = AST.Idents.get(Name, tok::TokenKind::identifier);
-
-    LookupResult Result(SemaRef, &II, SourceLocation(), Sema::LookupTagName);
-    CXXRecordDecl *PrevDecl = nullptr;
-    if (SemaRef.LookupQualifiedName(Result, HLSLNamespace)) {
-      // Declaration already exists (from precompiled headers)
-      NamedDecl *Found = Result.getFoundDecl();
-      if (auto *TD = dyn_cast<ClassTemplateDecl>(Found)) {
-        PrevDecl = TD->getTemplatedDecl();
-        PrevTemplate = TD;
-      } else
-        PrevDecl = dyn_cast<CXXRecordDecl>(Found);
-      assert(PrevDecl && "Unexpected lookup result type.");
-    }
-
-    if (PrevDecl && PrevDecl->isCompleteDefinition()) {
-      Record = PrevDecl;
-      Template = PrevTemplate;
-      return;
-    }
-
-    Record = CXXRecordDecl::Create(AST, TagDecl::TagKind::Class, HLSLNamespace,
-                                   SourceLocation(), SourceLocation(), &II,
-                                   PrevDecl, true);
-    Record->setImplicit(true);
-    Record->setLexicalDeclContext(HLSLNamespace);
-    Record->setHasExternalLexicalStorage();
-
-    // Don't let anyone derive from built-in types.
-    Record->addAttr(FinalAttr::CreateImplicit(AST, SourceRange(),
-                                              FinalAttr::Keyword_final));
-  }
-
-  ~BuiltinTypeDeclBuilder() {
-    if (HLSLNamespace && !Template && Record->getDeclContext() == HLSLNamespace)
-      HLSLNamespace->addDecl(Record);
-  }
-
-  CXXRecordDecl *finalizeForwardDeclaration() {
-    // Force the QualType to be generated for the record declaration. In most
-    // cases this will happen naturally when something uses the type the
-    // QualType gets lazily created. Unfortunately, with our injected types if a
-    // type isn't used in a translation unit the QualType may not get
-    // automatically generated before a PCH is generated. To resolve this we
-    // just force that the QualType is generated after we create a forward
-    // declaration.
-    (void)Record->getASTContext().getRecordType(Record);
-    return Record;
-  }
-
-  BuiltinTypeDeclBuilder &
-  addMemberVariable(StringRef Name, QualType Type, llvm::ArrayRef<Attr *> Attrs,
-                    AccessSpecifier Access = AccessSpecifier::AS_private) {
-    assert(!Record->isCompleteDefinition() && "record is already complete");
-    assert(Record->isBeingDefined() &&
-           "Definition must be started before adding members!");
-    ASTContext &AST = Record->getASTContext();
-
-    IdentifierInfo &II = AST.Idents.get(Name, tok::TokenKind::identifier);
-    TypeSourceInfo *MemTySource =
-        AST.getTrivialTypeSourceInfo(Type, SourceLocation());
-    auto *Field = FieldDecl::Create(
-        AST, Record, SourceLocation(), SourceLocation(), &II, Type, MemTySource,
-        nullptr, false, InClassInitStyle::ICIS_NoInit);
-    Field->setAccess(Access);
-    Field->setImplicit(true);
-    for (Attr *A : Attrs) {
-      if (A)
-        Field->addAttr(A);
-    }
-
-    Record->addDecl(Field);
-    Fields[Name] = Field;
-    return *this;
-  }
-
-  BuiltinTypeDeclBuilder &
-  addHandleMember(ResourceClass RC, bool IsROV, bool RawBuffer,
-                  AccessSpecifier Access = AccessSpecifier::AS_private) {
-    assert(!Record->isCompleteDefinition() && "record is already complete");
-
-    ASTContext &Ctx = SemaRef.getASTContext();
-    TypeSourceInfo *ElementTypeInfo =
-        Ctx.getTrivialTypeSourceInfo(getHandleElementType(), SourceLocation());
-
-    // add handle member with resource type attributes
-    QualType AttributedResTy = QualType();
-    SmallVector<const Attr *> Attrs = {
-        HLSLResourceClassAttr::CreateImplicit(Ctx, RC),
-        IsROV ? HLSLROVAttr::CreateImplicit(Ctx) : nullptr,
-        RawBuffer ? HLSLRawBufferAttr::CreateImplicit(Ctx) : nullptr,
-        ElementTypeInfo
-            ? HLSLContainedTypeAttr::CreateImplicit(Ctx, ElementTypeInfo)
-            : nullptr};
-    if (CreateHLSLAttributedResourceType(SemaRef, Ctx.HLSLResourceTy, Attrs,
-                                         AttributedResTy))
-      addMemberVariable("__handle", AttributedResTy, {}, Access);
-    return *this;
-  }
-
-  BuiltinTypeDeclBuilder &addDefaultHandleConstructor() {
-    if (Record->isCompleteDefinition())
-      return *this;
-    ASTContext &AST = Record->getASTContext();
-
-    QualType ConstructorType =
-        AST.getFunctionType(AST.VoidTy, {}, FunctionProtoType::ExtProtoInfo());
-
-    CanQualType CanTy = Record->getTypeForDecl()->getCanonicalTypeUnqualified();
-    DeclarationName Name = AST.DeclarationNames.getCXXConstructorName(CanTy);
-    CXXConstructorDecl *Constructor = CXXConstructorDecl::Create(
-        AST, Record, SourceLocation(),
-        DeclarationNameInfo(Name, SourceLocation()), ConstructorType,
-        AST.getTrivialTypeSourceInfo(ConstructorType, SourceLocation()),
-        ExplicitSpecifier(), false, true, false,
-        ConstexprSpecKind::Unspecified);
-
-    Constructor->setBody(CompoundStmt::Create(
-        AST, {}, FPOptionsOverride(), SourceLocation(), SourceLocation()));
-    Constructor->setAccess(AccessSpecifier::AS_public);
-    Record->addDecl(Constructor);
-    return *this;
-  }
-
-  BuiltinTypeDeclBuilder &addArraySubscriptOperators() {
-    ASTContext &AST = Record->getASTContext();
-    DeclarationName Subscript =
-        AST.DeclarationNames.getCXXOperatorName(OO_Subscript);
-
-    addHandleAccessFunction(Subscript, /*IsConst=*/true, /*IsRef=*/true);
-    addHandleAccessFunction(Subscript, /*IsConst=*/false, /*IsRef=*/true);
-    return *this;
-  }
-
-  BuiltinTypeDeclBuilder &addLoadMethods() {
-    if (Record->isCompleteDefinition())
-      return *this;
-
-    ASTContext &AST = Record->getASTContext();
-    IdentifierInfo &II = AST.Idents.get("Load", tok::TokenKind::identifier);
-    DeclarationName Load(&II);
-    // TODO: We also need versions with status for CheckAccessFullyMapped.
-    addHandleAccessFunction(Load, /*IsConst=*/false, /*IsRef=*/false);
-
-    return *this;
-  }
-
-  FieldDecl *getResourceHandleField() {
-    auto I = Fields.find("__handle");
-    assert(I != Fields.end() &&
-           I->second->getType()->isHLSLAttributedResourceType() &&
-           "record does not have resource handle field");
-    return I->second;
-  }
-
-  QualType getFirstTemplateTypeParam() {
-    assert(Template && "record it not a template");
-    if (const auto *TTD = dyn_cast<TemplateTypeParmDecl>(
-            Template->getTemplateParameters()->getParam(0))) {
-      return QualType(TTD->getTypeForDecl(), 0);
-    }
-    return QualType();
-  }
-
-  QualType getHandleElementType() {
-    if (Template)
-      return getFirstTemplateTypeParam();
-    // TODO: Should we default to VoidTy? Using `i8` is arguably ambiguous.
-    return SemaRef.getASTContext().Char8Ty;
-  }
-
-  BuiltinTypeDeclBuilder &startDefinition() {
-    assert(!Record->isCompleteDefinition() && "record is already complete");
-    Record->startDefinition();
-    return *this;
-  }
-
-  BuiltinTypeDeclBuilder &completeDefinition() {
-    assert(!Record->isCompleteDefinition() && "record is already complete");
-    assert(Record->isBeingDefined() &&
-           "Definition must be started before completing it.");
-
-    Record->completeDefinition();
-    return *this;
-  }
-
-  Expr *getConstantIntExpr(int value) {
-    ASTContext &AST = SemaRef.getASTContext();
-    return IntegerLiteral::Create(
-        AST, llvm::APInt(AST.getTypeSize(AST.IntTy), value, true), AST.IntTy,
-        SourceLocation());
-  }
-
-  TemplateParameterListBuilder addTemplateArgumentList();
-  BuiltinTypeDeclBuilder &addSimpleTemplateParams(ArrayRef<StringRef> Names,
-                                                  ConceptDecl *CD);
-
-  // Builtin types methods
-  BuiltinTypeDeclBuilder &addIncrementCounterMethod();
-  BuiltinTypeDeclBuilder &addDecrementCounterMethod();
-  BuiltinTypeDeclBuilder &addHandleAccessFunction(DeclarationName &Name,
-                                                  bool IsConst, bool IsRef);
-  BuiltinTypeDeclBuilder &addAppendMethod();
-  BuiltinTypeDeclBuilder &addConsumeMethod();
-};
-
-struct TemplateParameterListBuilder {
-  BuiltinTypeDeclBuilder &Builder;
-  llvm::SmallVector<NamedDecl *> Params;
-
-  TemplateParameterListBuilder(BuiltinTypeDeclBuilder &RB) : Builder(RB) {}
-
-  ~TemplateParameterListBuilder() { finalizeTemplateArgs(); }
-
-  TemplateParameterListBuilder &
-  addTypeParameter(StringRef Name, QualType DefaultValue = QualType()) {
-    assert(!Builder.Record->isCompleteDefinition() &&
-           "record is already complete");
-    ASTContext &AST = Builder.SemaRef.getASTContext();
-    unsigned Position = static_cast<unsigned>(Params.size());
-    auto *Decl = TemplateTypeParmDecl::Create(
-        AST, Builder.Record->getDeclContext(), SourceLocation(),
-        SourceLocation(), /* TemplateDepth */ 0, Position,
-        &AST.Idents.get(Name, tok::TokenKind::identifier),
-        /* Typename */ true,
-        /* ParameterPack */ false,
-        /* HasTypeConstraint*/ false);
-    if (!DefaultValue.isNull())
-      Decl->setDefaultArgument(AST,
-                               Builder.SemaRef.getTrivialTemplateArgumentLoc(
-                                   DefaultValue, QualType(), SourceLocation()));
-
-    Params.emplace_back(Decl);
-    return *this;
-  }
-
-  // The concept specialization expression (CSE) constructed in
-  // constructConceptSpecializationExpr is constructed so that it
-  // matches the CSE that is constructed when parsing the below C++ code:
-  //
-  // template<typename T>
-  // concept is_typed_resource_element_compatible =
-  // __builtin_hlsl_typed_resource_element_compatible<T>
-  //
-  // template<typename element_type> requires
-  // is_typed_resource_element_compatible<element_type>
-  // struct RWBuffer {
-  //     element_type Val;
-  // };
-  //
-  // int fn() {
-  //     RWBuffer<int> Buf;
-  // }
-  //
-  // When dumping the AST and filtering for "RWBuffer", the resulting AST
-  // structure is what we're trying to construct below, specifically the
-  // CSE portion.
-  ConceptSpecializationExpr *
-  constructConceptSpecializationExpr(Sema &S, ConceptDecl *CD) {
-    ASTContext &Context = S.getASTContext();
-    SourceLocation Loc = Builder.Record->getBeginLoc();
-    DeclarationNameInfo DNI(CD->getDeclName(), Loc);
-    NestedNameSpecifierLoc NNSLoc;
-    DeclContext *DC = Builder.Record->getDeclContext();
-    TemplateArgumentListInfo TALI(Loc, Loc);
-
-    // Assume that the concept decl has just one template parameter
-    // This parameter should have been added when CD was constructed
-    // in getTypedBufferConceptDecl
-    assert(CD->getTemplateParameters()->size() == 1 &&
-           "unexpected concept decl parameter count");
-    TemplateTypeParmDecl *ConceptTTPD = dyn_cast<TemplateTypeParmDecl>(
-        CD->getTemplateParameters()->getParam(0));
-
-    // this TemplateTypeParmDecl is the template for the resource, and is
-    // used to construct a template argumentthat will be used
-    // to construct the ImplicitConceptSpecializationDecl
-    TemplateTypeParmDecl *T = TemplateTypeParmDecl::Create(
-        Context,                          // AST context
-        Builder.Record->getDeclContext(), // DeclContext
-        SourceLocation(), SourceLocation(),
-        /*D=*/0,                    // Depth in the template parameter list
-        /*P=*/0,                    // Position in the template parameter list
-        /*Id=*/nullptr,             // Identifier for 'T'
-        /*Typename=*/true,          // Indicates this is a 'typename' or 'class'
-        /*ParameterPack=*/false,    // Not a parameter pack
-        /*HasTypeConstraint=*/false // Has no type constraint
-    );
-
-    T->setDeclContext(DC);
-
-    QualType ConceptTType = Context.getTypeDeclType(ConceptTTPD);
-
-    // this is the 2nd template argument node, on which
-    // the concept constraint is actually being applied: 'element_type'
-    TemplateArgument ConceptTA = TemplateArgument(ConceptTType);
-
-    QualType CSETType = Context.getTypeDeclType(T);
-
-    // this is the 1st template argument node, which represents
-    // the abstract type that a concept would refer to: 'T'
-    TemplateArgument CSETA = TemplateArgument(CSETType);
-
-    ImplicitConceptSpecializationDecl *ImplicitCSEDecl =
-        ImplicitConceptSpecializationDecl::Create(
-            Context, Builder.Record->getDeclContext(), Loc, {CSETA});
-
-    // Constraint satisfaction is used to construct the
-    // ConceptSpecailizationExpr, and represents the 2nd Template Argument,
-    // located at the bottom of the sample AST above.
-    const ConstraintSatisfaction CS(CD, {ConceptTA});
-    TemplateArgumentLoc TAL = S.getTrivialTemplateArgumentLoc(
-        ConceptTA, QualType(), SourceLocation());
-
-    TALI.addArgument(TAL);
-    const ASTTemplateArgumentListInfo *ATALI =
-        ASTTemplateArgumentListInfo::Create(Context, TALI);
-
-    // In the concept reference, ATALI is what adds the extra
-    // TemplateArgument node underneath CSE
-    ConceptReference *CR =
-        ConceptReference::Create(Context, NNSLoc, Loc, DNI, CD, CD, ATALI);
-
-    ConceptSpecializationExpr *CSE =
-        ConceptSpecializationExpr::Create(Context, CR, ImplicitCSEDecl, &CS);
-
-    return CSE;
-  }
-
-  BuiltinTypeDeclBuilder &finalizeTemplateArgs(ConceptDecl *CD = nullptr) {
-    if (Params.empty())
-      return Builder;
-
-    ASTContext &AST = Builder.SemaRef.Context;
-    ConceptSpecializationExpr *CSE =
-        CD ? constructConceptSpecializationExpr(Builder.SemaRef, CD) : nullptr;
-    auto *ParamList = TemplateParameterList::Create(
-        AST, SourceLocation(), SourceLocation(), Params, SourceLocation(), CSE);
-    Builder.Template = ClassTemplateDecl::Create(
-        AST, Builder.Record->getDeclContext(), SourceLocation(),
-        DeclarationName(Builder.Record->getIdentifier()), ParamList,
-        Builder.Record);
-
-    Builder.Record->setDescribedClassTemplate(Builder.Template);
-    Builder.Template->setImplicit(true);
-    Builder.Template->setLexicalDeclContext(Builder.Record->getDeclContext());
-
-    // NOTE: setPreviousDecl before addDecl so new decl replace old decl when
-    // make visible.
-    Builder.Template->setPreviousDecl(Builder.PrevTemplate);
-    Builder.Record->getDeclContext()->addDecl(Builder.Template);
-    Params.clear();
-
-    QualType T = Builder.Template->getInjectedClassNameSpecialization();
-    T = AST.getInjectedClassNameType(Builder.Record, T);
-
-    return Builder;
-  }
-};
-
-// Builder for methods of builtin types. Allows adding methods to builtin types
-// using the builder pattern like this:
-//
-//   BuiltinTypeMethodBuilder(RecordBuilder, "MethodName", ReturnType)
-//       .addParam("param_name", Type, InOutModifier)
-//       .callBuiltin("builtin_name", BuiltinParams...)
-//       .finalizeMethod();
-//
-// The builder needs to have all of the method parameters before it can create
-// a CXXMethodDecl. It collects them in addParam calls and when a first
-// method that builds the body is called or when access to 'this` is needed it
-// creates the CXXMethodDecl and ParmVarDecls instances. These can then be
-// referenced from the body building methods. Destructor or an explicit call to
-// finalizeMethod() will complete the method definition.
-//
-// The callBuiltin helper method accepts constants via `Expr *` or placeholder
-// value arguments to indicate which function arguments to forward to the
-// builtin.
-//
-// If the method that is being built has a non-void return type the
-// finalizeMethod will create a return statent with the value of the last
-// statement (unless the last statement is already a ReturnStmt).
-struct BuiltinTypeMethodBuilder {
-  struct MethodParam {
-    const IdentifierInfo &NameII;
-    QualType Ty;
-    HLSLParamModifierAttr::Spelling Modifier;
-    MethodParam(const IdentifierInfo &NameII, QualType Ty,
-                HLSLParamModifierAttr::Spelling Modifier)
-        : NameII(NameII), Ty(Ty), Modifier(Modifier) {}
-  };
-
-  BuiltinTypeDeclBuilder &DeclBuilder;
-  DeclarationNameInfo NameInfo;
-  QualType ReturnTy;
-  CXXMethodDecl *Method;
-  bool IsConst;
-  llvm::SmallVector<MethodParam> Params;
-  llvm::SmallVector<Stmt *> StmtsList;
-
-  // Argument placeholders, inspired by std::placeholder. These are the indices
-  // of arguments to forward to `callBuiltin` and other method builder methods.
-  // Additional special values are:
-  //   Handle   - refers to the resource handle.
-  //   LastStmt - refers to the last statement in the method body; referencing
-  //              LastStmt will remove the statement from the method body since
-  //              it will be linked from the new expression being constructed.
-  enum class PlaceHolder { _0, _1, _2, _3, Handle = 128, LastStmt };
-
-  Expr *convertPlaceholder(PlaceHolder PH) {
-    if (PH == PlaceHolder::Handle)
-      return getResourceHandleExpr();
-
-    if (PH == PlaceHolder::LastStmt) {
-      assert(!StmtsList.empty() && "no statements in the list");
-      Stmt *LastStmt = StmtsList.pop_back_val();
-      assert(isa<ValueStmt>(LastStmt) &&
-             "last statement does not have a value");
-      return cast<ValueStmt>(LastStmt)->getExprStmt();
-    }
-
-    ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
-    ParmVarDecl *ParamDecl = Method->getParamDecl(static_cast<unsigned>(PH));
-    return DeclRefExpr::Create(
-        AST, NestedNameSpecifierLoc(), SourceLocation(), ParamDecl, false,
-        DeclarationNameInfo(ParamDecl->getDeclName(), SourceLocation()),
-        ParamDecl->getType(), VK_PRValue);
-  }
-  Expr *convertPlaceholder(Expr *E) { return E; }
-
-public:
-  BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB, DeclarationName &Name,
-                           QualType ReturnTy, bool IsConst = false)
-      : DeclBuilder(DB), NameInfo(DeclarationNameInfo(Name, SourceLocation())),
-        ReturnTy(ReturnTy), Method(nullptr), IsConst(IsConst) {}
-
-  BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB, StringRef Name,
-                           QualType ReturnTy, bool IsConst = false)
-      : DeclBuilder(DB), ReturnTy(ReturnTy), Method(nullptr), IsConst(IsConst) {
-    const IdentifierInfo &II =
-        DB.SemaRef.getASTContext().Idents.get(Name, tok::TokenKind::identifier);
-    NameInfo = DeclarationNameInfo(DeclarationName(&II), SourceLocation());
-  }
-
-  BuiltinTypeMethodBuilder &addParam(StringRef Name, QualType Ty,
-                                     HLSLParamModifierAttr::Spelling Modifier =
-                                         HLSLParamModifierAttr::Keyword_in) {
-    assert(Method == nullptr && "Cannot add param, method already created");
-    const IdentifierInfo &II = DeclBuilder.SemaRef.getASTContext().Idents.get(
-        Name, tok::TokenKind::identifier);
-    Params.emplace_back(II, Ty, Modifier);
-    return *this;
-  }
-
-private:
-  void createMethodDecl() {
-    assert(Method == nullptr && "Method already created");
-
-    // create method type
-    ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
-    SmallVector<QualType> ParamTypes;
-    for (MethodParam &MP : Params)
-      ParamTypes.emplace_back(MP.Ty);
-
-    FunctionProtoType::ExtProtoInfo ExtInfo;
-    if (IsConst)
-      ExtInfo.TypeQuals.addConst();
-
-    QualType MethodTy = AST.getFunctionType(ReturnTy, ParamTypes, ExtInfo);
-
-    // create method decl
-    auto *TSInfo = AST.getTrivialTypeSourceInfo(MethodTy, SourceLocation());
-    Method =
-        CXXMethodDecl::Create(AST, DeclBuilder.Record, SourceLocation(),
-                              NameInfo, MethodTy, TSInfo, SC_None, false, false,
-                              ConstexprSpecKind::Unspecified, SourceLocation());
-
-    // create params & set them to the function prototype
-    SmallVector<ParmVarDecl *> ParmDecls;
-    auto FnProtoLoc =
-        Method->getTypeSourceInfo()->getTypeLoc().getAs<FunctionProtoTypeLoc>();
-    for (int I = 0, E = Params.size(); I != E; I++) {
-      MethodParam &MP = Params[I];
-      ParmVarDecl *Parm = ParmVarDecl::Create(
-          AST, Method->getDeclContext(), SourceLocation(), SourceLocation(),
-          &MP.NameII, MP.Ty,
-          AST.getTrivialTypeSourceInfo(MP.Ty, SourceLocation()), SC_None,
-          nullptr);
-      if (MP.Modifier != HLSLParamModifierAttr::Keyword_in) {
-        auto *Mod =
-            HLSLParamModifierAttr::Create(AST, SourceRange(), MP.Modifier);
-        Parm->addAttr(Mod);
-      }
-      ParmDecls.push_back(Parm);
-      FnProtoLoc.setParam(I, Parm);
-    }
-    Method->setParams({ParmDecls});
-  }
-
-public:
-  ~BuiltinTypeMethodBuilder() { finalizeMethod(); }
-
-  BuiltinTypeMethodBuilder(const BuiltinTypeMethodBuilder &Other) = delete;
-  BuiltinTypeMethodBuilder &
-  operator=(const BuiltinTypeMethodBuilder &Other) = delete;
-
-  Expr *getResourceHandleExpr() {
-    // The first statement added to a method or access to 'this' creates the
-    // declaration.
-    if (!Method)
-      createMethodDecl();
-
-    ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
-    CXXThisExpr *This = CXXThisExpr::Create(
-        AST, SourceLocation(), Method->getFunctionObjectParameterType(), true);
-    FieldDecl *HandleField = DeclBuilder.getResourceHandleField();
-    return MemberExpr::CreateImplicit(AST, This, false, HandleField,
-                                      HandleField->getType(), VK_LValue,
-                                      OK_Ordinary);
-  }
-
-  template <typename... Ts>
-  BuiltinTypeMethodBuilder &callBuiltin(StringRef BuiltinName,
-                                        QualType ReturnType, Ts... ArgSpecs) {
-    std::array<Expr *, sizeof...(ArgSpecs)> Args{
-        convertPlaceholder(std::forward<Ts>(ArgSpecs))...};
-
-    // The first statement added to a method or access to 'this` creates the
-    // declaration.
-    if (!Method)
-      createMethodDecl();
-
-    ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
-    FunctionDecl *FD = lookupBuiltinFunction(DeclBuilder.SemaRef, BuiltinName);
-    DeclRefExpr *DRE = DeclRefExpr::Create(
-        AST, NestedNameSpecifierLoc(), SourceLocation(), FD, false,
-        FD->getNameInfo(), AST.BuiltinFnTy, VK_PRValue);
-
-    if (ReturnType.isNull())
-      ReturnType = FD->getReturnType();
-
-    Expr *Call = CallExpr::Create(AST, DRE, Args, ReturnType, VK_PRValue,
-                                  SourceLocation(), FPOptionsOverride());
-    StmtsList.push_back(Call);
-    return *this;
-  }
-
-  template <typename TLHS, typename TRHS>
-  BuiltinTypeMethodBuilder &assign(TLHS LHS, TRHS RHS) {
-    Expr *LHSExpr = convertPlaceholder(LHS);
-    Expr *RHSExpr = convertPlaceholder(RHS);
-    Stmt *AssignStmt = BinaryOperator::Create(
-        DeclBuilder.SemaRef.getASTContext(), LHSExpr, RHSExpr, BO_Assign,
-        LHSExpr->getType(), ExprValueKind::VK_PRValue,
-        ExprObjectKind::OK_Ordinary, SourceLocation(), FPOptionsOverride());
-    StmtsList.push_back(AssignStmt);
-    return *this;
-  }
-
-  template <typename T> BuiltinTypeMethodBuilder &dereference(T Ptr) {
-    Expr *PtrExpr = convertPlaceholder(Ptr);
-    Expr *Deref =
-        UnaryOperator::Create(DeclBuilder.SemaRef.getASTContext(), PtrExpr,
-                              UO_Deref, PtrExpr->getType()->getPointeeType(),
-                              VK_PRValue, OK_Ordinary, SourceLocation(),
-                              /*CanOverflow=*/false, FPOptionsOverride());
-    StmtsList.push_back(Deref);
-    return *this;
-  }
-
-  BuiltinTypeDeclBuilder &finalizeMethod() {
-    assert(!DeclBuilder.Record->isCompleteDefinition() &&
-           "record is already complete");
-    assert(
-        Method != nullptr &&
-        "method decl not created; are you missing a call to build the body?");
-
-    if (!Method->hasBody()) {
-      ASTContext &AST = DeclBuilder.SemaRef.getASTContext();
-      assert((ReturnTy == AST.VoidTy || !StmtsList.empty()) &&
-             "nothing to return from non-void method");
-      if (ReturnTy != AST.VoidTy) {
-        if (Expr *LastExpr = dyn_cast<Expr>(StmtsList.back())) {
-          assert(AST.hasSameUnqualifiedType(LastExpr->getType(),
-                                            ReturnTy.getNonReferenceType()) &&
-                 "Return type of the last statement must match the return type "
-                 "of the method");
-          if (!isa<ReturnStmt>(LastExpr)) {
-            StmtsList.pop_back();
-            StmtsList.push_back(
-                ReturnStmt::Create(AST, SourceLocation(), LastExpr, nullptr));
-          }
-        }
-      }
-
-      Method->setBody(CompoundStmt::Create(AST, StmtsList, FPOptionsOverride(),
-                                           SourceLocation(), SourceLocation()));
-      Method->setLexicalDeclContext(DeclBuilder.Record);
-      Method->setAccess(AccessSpecifier::AS_public);
-      Method->addAttr(AlwaysInlineAttr::CreateImplicit(
-          AST, SourceRange(), AlwaysInlineAttr::CXX11_clang_always_inline));
-      DeclBuilder.Record->addDecl(Method);
-    }
-    return DeclBuilder;
-  }
-};
-
-} // namespace
-
-TemplateParameterListBuilder BuiltinTypeDeclBuilder::addTemplateArgumentList() {
-  return TemplateParameterListBuilder(*this);
-}
-
-BuiltinTypeDeclBuilder &
-BuiltinTypeDeclBuilder::addSimpleTemplateParams(ArrayRef<StringRef> Names,
-                                                ConceptDecl *CD = nullptr) {
-  if (Record->isCompleteDefinition()) {
-    assert(Template && "existing record it not a template");
-    assert(Template->getTemplateParameters()->size() == Names.size() &&
-           "template param count mismatch");
-    return *this;
-  }
-
-  TemplateParameterListBuilder Builder = this->addTemplateArgumentList();
-  for (StringRef Name : Names)
-    Builder.addTypeParameter(Name);
-  return Builder.finalizeTemplateArgs(CD);
-}
-
-BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addIncrementCounterMethod() {
-  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
-  return BuiltinTypeMethodBuilder(*this, "IncrementCounter",
-                                  SemaRef.getASTContext().UnsignedIntTy)
-      .callBuiltin("__builtin_hlsl_buffer_update_counter", QualType(),
-                   PH::Handle, getConstantIntExpr(1))
-      .finalizeMethod();
-}
-
-BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addDecrementCounterMethod() {
-  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
-  return BuiltinTypeMethodBuilder(*this, "DecrementCounter",
-                                  SemaRef.getASTContext().UnsignedIntTy)
-      .callBuiltin("__builtin_hlsl_buffer_update_counter", QualType(),
-                   PH::Handle, getConstantIntExpr(-1))
-      .finalizeMethod();
-}
-
-BuiltinTypeDeclBuilder &
-BuiltinTypeDeclBuilder::addHandleAccessFunction(DeclarationName &Name,
-                                                bool IsConst, bool IsRef) {
-  assert(!Record->isCompleteDefinition() && "record is already complete");
-  ASTContext &AST = SemaRef.getASTContext();
-  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
-
-  QualType ElemTy = getHandleElementType();
-  // TODO: Map to an hlsl_device address space.
-  QualType ElemPtrTy = AST.getPointerType(ElemTy);
-  QualType ReturnTy = ElemTy;
-  if (IsConst)
-    ReturnTy.addConst();
-  if (IsRef)
-    ReturnTy = AST.getLValueReferenceType(ReturnTy);
-
-  return BuiltinTypeMethodBuilder(*this, Name, ReturnTy, IsConst)
-      .addParam("Index", AST.UnsignedIntTy)
-      .callBuiltin("__builtin_hlsl_resource_getpointer", ElemPtrTy, PH::Handle,
-                   PH::_0)
-      .dereference(PH::LastStmt)
-      .finalizeMethod();
-}
-
-BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addAppendMethod() {
-  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
-  ASTContext &AST = SemaRef.getASTContext();
-  QualType ElemTy = getHandleElementType();
-  return BuiltinTypeMethodBuilder(*this, "Append", AST.VoidTy)
-      .addParam("value", ElemTy)
-      .callBuiltin("__builtin_hlsl_buffer_update_counter", AST.UnsignedIntTy,
-                   PH::Handle, getConstantIntExpr(1))
-      .callBuiltin("__builtin_hlsl_resource_getpointer",
-                   AST.getPointerType(ElemTy), PH::Handle, PH::LastStmt)
-      .dereference(PH::LastStmt)
-      .assign(PH::LastStmt, PH::_0)
-      .finalizeMethod();
-}
-
-BuiltinTypeDeclBuilder &BuiltinTypeDeclBuilder::addConsumeMethod() {
-  using PH = BuiltinTypeMethodBuilder::PlaceHolder;
-  ASTContext &AST = SemaRef.getASTContext();
-  QualType ElemTy = getHandleElementType();
-  return BuiltinTypeMethodBuilder(*this, "Consume", ElemTy)
-      .callBuiltin("__builtin_hlsl_buffer_update_counter", AST.UnsignedIntTy,
-                   PH::Handle, getConstantIntExpr(-1))
-      .callBuiltin("__builtin_hlsl_resource_getpointer",
-                   AST.getPointerType(ElemTy), PH::Handle, PH::LastStmt)
-      .dereference(PH::LastStmt)
-      .finalizeMethod();
-}
-
-HLSLExternalSemaSource::~HLSLExternalSemaSource() {}
+using clang::hlsl::BuiltinTypeDeclBuilder;
 
 void HLSLExternalSemaSource::InitializeSema(Sema &S) {
   SemaPtr = &S;
@@ -1122,19 +393,3 @@ void HLSLExternalSemaSource::CompleteType(TagDecl *Tag) {
     return;
   It->second(Record);
 }
-
-static FunctionDecl *lookupBuiltinFunction(Sema &S, StringRef Name) {
-  IdentifierInfo &II =
-      S.getASTContext().Idents.get(Name, tok::TokenKind::identifier);
-  DeclarationNameInfo NameInfo =
-      DeclarationNameInfo(DeclarationName(&II), SourceLocation());
-  LookupResult R(S, NameInfo, Sema::LookupOrdinaryName);
-  // AllowBuiltinCreation is false but LookupDirect will create
-  // the builtin when searching the global scope anyways...
-  S.LookupName(R, S.getCurScope());
-  // FIXME: If the builtin function was user-declared in global scope,
-  // this assert *will* fail. Should this call LookupBuiltin instead?
-  assert(R.isSingleResult() &&
-         "Since this is a builtin it should always resolve!");
-  return cast<FunctionDecl>(R.getFoundDecl());
-}
diff --git a/clang/test/AST/HLSL/HLSLControlFlowHint.hlsl b/clang/test/AST/HLSL/HLSLControlFlowHint.hlsl
index c1e6d969c8d31..18263bedbe3ec 100644
--- a/clang/test/AST/HLSL/HLSLControlFlowHint.hlsl
+++ b/clang/test/AST/HLSL/HLSLControlFlowHint.hlsl
@@ -41,3 +41,62 @@ export int no_attr(int X){
 
     return resp;
 }
+
+// CHECK: FunctionDecl {{.*}} used flatten_switch 'int (int)'
+// CHECK: AttributedStmt
+// CHECK-NEXT: HLSLControlFlowHintAttr {{.*}} flatten
+export int flatten_switch(int X){
+    int resp;
+    [flatten] 
+    switch (X) {
+        case 0:
+        resp = -X;
+        break;
+    case 1:
+         resp = X+X;
+        break;
+    case 2:
+        resp = X * X; break;
+    }
+
+    return resp;
+}
+
+// CHECK: FunctionDecl {{.*}} used branch_switch 'int (int)'
+// CHECK: AttributedStmt
+// CHECK-NEXT: HLSLControlFlowHintAttr {{.*}} branch
+export int branch_switch(int X){
+    int resp;
+    [branch] 
+    switch (X) {
+     case 0:
+        resp = -X;
+        break;
+    case 1:
+        resp = X+X;
+        break;
+    case 2:
+        resp = X * X; break;
+    }
+
+    return resp;
+}
+
+// CHECK: FunctionDecl {{.*}} used no_attr_switch 'int (int)'
+// CHECK-NOT: AttributedStmt
+// CHECK-NOT: HLSLControlFlowHintAttr
+export int no_attr_switch(int X){
+    int resp;
+    switch (X) {
+        case 0:
+        resp = -X;
+        break;
+    case 1:
+         resp = X+X;
+        break;
+    case 2:
+        resp = X * X; break;
+    }
+
+    return resp;
+}
diff --git a/clang/test/Driver/hip-code-object-version.hip b/clang/test/Driver/hip-code-object-version.hip
index 30d8644dff54c..6767c27761d0a 100644
--- a/clang/test/Driver/hip-code-object-version.hip
+++ b/clang/test/Driver/hip-code-object-version.hip
@@ -27,12 +27,10 @@
 // RUN:   --offload-arch=gfx906 --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=V6 %s
 
-// V6: warning: code object v6 is still in development and not ready for production use yet; use at your own risk
 // V6: "-mcode-object-version=6"
 // V6: "-mllvm" "--amdhsa-code-object-version=6"
 // V6: "-targets=host-x86_64-unknown-linux-gnu,hipv4-amdgcn-amd-amdhsa--gfx906"
 
-
 // Check bundle ID for code object version default
 
 // RUN: %clang -### --target=x86_64-linux-gnu \
diff --git a/clang/test/Misc/warning-flags.c b/clang/test/Misc/warning-flags.c
index a978835a41014..7ac77cc91fdc1 100644
--- a/clang/test/Misc/warning-flags.c
+++ b/clang/test/Misc/warning-flags.c
@@ -18,7 +18,7 @@ This test serves two purposes:
 
 The list of warnings below should NEVER grow.  It should gradually shrink to 0.
 
-CHECK: Warnings without flags (58):
+CHECK: Warnings without flags (57):
 
 CHECK-NEXT:   ext_expected_semi_decl_list
 CHECK-NEXT:   ext_missing_whitespace_after_macro_name
@@ -40,7 +40,6 @@ CHECK-NEXT:   warn_collection_expr_type
 CHECK-NEXT:   warn_conflicting_variadic
 CHECK-NEXT:   warn_delete_array_type
 CHECK-NEXT:   warn_double_const_requires_fp64
-CHECK-NEXT:   warn_drv_amdgpu_cov6
 CHECK-NEXT:   warn_drv_assuming_mfloat_abi_is
 CHECK-NEXT:   warn_drv_clang_unsupported
 CHECK-NEXT:   warn_drv_pch_not_first_include
diff --git a/clang/unittests/ASTMatchers/ASTMatchersInternalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersInternalTest.cpp
index 539a5877a8c4f..a930638f355b9 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersInternalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersInternalTest.cpp
@@ -198,7 +198,7 @@ TEST(AstPolymorphicMatcherPMacro, Works) {
 }
 
 TEST(MatchFinder, CheckProfiling) {
-  MatchFinderOptions Options;
+  MatchFinder::MatchFinderOptions Options;
   llvm::StringMap<llvm::TimeRecord> Records;
   Options.CheckProfiling.emplace(Records);
   MatchFinder Finder(std::move(Options));
diff --git a/compiler-rt/lib/asan/asan_shadow_setup.cpp b/compiler-rt/lib/asan/asan_shadow_setup.cpp
index e66b8af1d2c30..5b3591da067bd 100644
--- a/compiler-rt/lib/asan/asan_shadow_setup.cpp
+++ b/compiler-rt/lib/asan/asan_shadow_setup.cpp
@@ -109,12 +109,14 @@ void InitializeShadowMemory() {
     ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
     ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
   } else {
-    // The shadow mappings can shadow the entire user address space. However,
-    // on 32-bit systems, the maximum ASLR entropy (currently up to 16-bits
-    // == 256MB) is a significant chunk of the address space; reclaiming it by
-    // disabling ASLR might allow chonky binaries to run.
-    if (sizeof(uptr) == 32)
-      TryReExecWithoutASLR();
+    // ASan's mappings can usually shadow the entire address space, even with
+    // maximum ASLR entropy. However:
+    // - On 32-bit systems, the maximum ASLR entropy (currently up to 16-bits
+    //   == 256MB) is a significant chunk of the address space; reclaiming it
+    //   by disabling ASLR might allow chonky binaries to run.
+    // - On 64-bit systems, some settings (e.g., for Linux, unlimited stack
+    //   size plus 31+ bits of entropy) can lead to an incompatible layout.
+    TryReExecWithoutASLR();
 
     Report(
         "Shadow memory range interleaves with an existing memory mapping. "
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index f671c8167a3c4..6d79b80593379 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -892,10 +892,9 @@ constexpr u32 kGuardWaiter = 1 << 17;
 
 static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g,
                          bool blocking_hooks = true) {
-  if (blocking_hooks)
-    OnPotentiallyBlockingRegionBegin();
-  auto on_exit = at_scope_exit([blocking_hooks] {
-    if (blocking_hooks)
+  bool in_potentially_blocking_region = false;
+  auto on_exit = at_scope_exit([&] {
+    if (in_potentially_blocking_region)
       OnPotentiallyBlockingRegionEnd();
   });
 
@@ -912,8 +911,13 @@ static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g,
     } else {
       if ((cmp & kGuardWaiter) ||
           atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter,
-                                         memory_order_relaxed))
+                                         memory_order_relaxed)) {
+        if (blocking_hooks && !in_potentially_blocking_region) {
+          in_potentially_blocking_region = true;
+          OnPotentiallyBlockingRegionBegin();
+        }
         FutexWait(g, cmp | kGuardWaiter);
+      }
     }
   }
 }
diff --git a/compiler-rt/test/tsan/cxa_guard_acquire.cpp b/compiler-rt/test/tsan/cxa_guard_acquire.cpp
index 100a40b281410..fc407259e8968 100644
--- a/compiler-rt/test/tsan/cxa_guard_acquire.cpp
+++ b/compiler-rt/test/tsan/cxa_guard_acquire.cpp
@@ -1,31 +1,95 @@
 // RUN: %clangxx_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
 
+#include "test.h"
+#include <assert.h>
 #include <sanitizer/tsan_interface.h>
 #include <stdio.h>
 
+// We only enter a potentially blocking region on thread contention. To reliably
+// trigger this, we force the initialization function to block until another
+// thread has entered the potentially blocking region.
+
+static bool init_done = false;
+
 namespace __tsan {
 
 #if (__APPLE__)
 __attribute__((weak))
 #endif
 void OnPotentiallyBlockingRegionBegin() {
-  printf("Enter __cxa_guard_acquire\n");
+  assert(!init_done);
+  printf("Enter potentially blocking region\n");
+  // Signal the other thread to finish initialization.
+  barrier_wait(&barrier);
 }
 
 #if (__APPLE__)
 __attribute__((weak))
 #endif
-void OnPotentiallyBlockingRegionEnd() { printf("Exit __cxa_guard_acquire\n"); }
+void OnPotentiallyBlockingRegionEnd() {
+  printf("Exit potentially blocking region\n");
+}
 
 } // namespace __tsan
 
+struct LazyInit {
+  LazyInit() {
+    assert(!init_done);
+    printf("Enter constructor\n");
+    // Wait for the other thread to get to the blocking region.
+    barrier_wait(&barrier);
+    printf("Exit constructor\n");
+  }
+};
+
+const LazyInit &get_lazy_init() {
+  static const LazyInit lazy_init;
+  return lazy_init;
+}
+
+void *thread(void *arg) {
+  get_lazy_init();
+  return nullptr;
+}
+
+struct LazyInit2 {
+  LazyInit2() { printf("Enter constructor 2\n"); }
+};
+
+const LazyInit2 &get_lazy_init2() {
+  static const LazyInit2 lazy_init2;
+  return lazy_init2;
+}
+
 int main(int argc, char **argv) {
   // CHECK: Enter main
   printf("Enter main\n");
-  // CHECK-NEXT: Enter __cxa_guard_acquire
-  // CHECK-NEXT: Exit __cxa_guard_acquire
-  static int s = argc;
-  (void)s;
+
+  // If initialization is contended, the blocked thread should enter a
+  // potentially blocking region.
+  //
+  // CHECK-NEXT: Enter constructor
+  // CHECK-NEXT: Enter potentially blocking region
+  // CHECK-NEXT: Exit constructor
+  // CHECK-NEXT: Exit potentially blocking region
+  barrier_init(&barrier, 2);
+  pthread_t th1, th2;
+  pthread_create(&th1, nullptr, thread, nullptr);
+  pthread_create(&th2, nullptr, thread, nullptr);
+  pthread_join(th1, nullptr);
+  pthread_join(th2, nullptr);
+
+  // Now that the value has been initialized, subsequent calls should not enter
+  // a potentially blocking region.
+  init_done = true;
+  get_lazy_init();
+
+  // If uncontended, there is no potentially blocking region.
+  //
+  // CHECK-NEXT: Enter constructor 2
+  get_lazy_init2();
+  get_lazy_init2();
+
   // CHECK-NEXT: Exit main
   printf("Exit main\n");
   return 0;
diff --git a/flang/cmake/modules/FlangCommon.cmake b/flang/cmake/modules/FlangCommon.cmake
index 4726c640c97b7..9dbd1959fe7b1 100644
--- a/flang/cmake/modules/FlangCommon.cmake
+++ b/flang/cmake/modules/FlangCommon.cmake
@@ -24,30 +24,16 @@ if (FLANG_RUNTIME_F128_MATH_LIB)
   add_compile_definitions(FLANG_RUNTIME_F128_MATH_LIB="${FLANG_RUNTIME_F128_MATH_LIB}")
 endif()
 
-# The NVPTX target can't emit a binary due to the PTXAS dependency, just
-# hard-code this.
-if ("${LLVM_RUNTIMES_TARGET}" MATCHES "^nvptx")
-  add_compile_definitions(FLANG_LITTLE_ENDIAN=1)
-else ()
-  # Check if 128-bit float computations can be done via long double
-  # Note that '-nostdinc++' might be implied when this code kicks in
-  # (see 'runtimes/CMakeLists.txt'), so we cannot use 'cfloat' C++ header
-  # file in the test below.
-  # Compile it as C.
-  check_c_source_compiles(
-    "#include <float.h>
-     #if LDBL_MANT_DIG != 113
-     #error LDBL_MANT_DIG != 113
-     #endif
-     int main() { return 0; }
-    "
-    HAVE_LDBL_MANT_DIG_113)
-
-  include(TestBigEndian)
-  test_big_endian(IS_BIGENDIAN)
-  if (IS_BIGENDIAN)
-    add_compile_definitions(FLANG_BIG_ENDIAN=1)
-  else ()
-    add_compile_definitions(FLANG_LITTLE_ENDIAN=1)
-  endif ()
-endif ()
+# Check if 128-bit float computations can be done via long double
+# Note that '-nostdinc++' might be implied when this code kicks in
+# (see 'runtimes/CMakeLists.txt'), so we cannot use 'cfloat' C++ header
+# file in the test below.
+# Compile it as C.
+check_c_source_compiles(
+  "#include <float.h>
+   #if LDBL_MANT_DIG != 113
+   #error LDBL_MANT_DIG != 113
+   #endif
+   int main() { return 0; }
+  "
+  HAVE_LDBL_MANT_DIG_113)
diff --git a/flang/include/flang/Common/api-attrs.h b/flang/include/flang/Common/api-attrs.h
index 1ee91ca8e0d9d..fd524ee34ccff 100644
--- a/flang/include/flang/Common/api-attrs.h
+++ b/flang/include/flang/Common/api-attrs.h
@@ -189,4 +189,21 @@
 #define RT_OPTNONE_ATTR
 #endif
 
+/* Detect system endianness if it was not explicitly set. */
+#if !defined(FLANG_LITTLE_ENDIAN) && !defined(FLANG_BIG_ENDIAN)
+
+/* We always assume Windows is little endian, otherwise use the GCC compatible
+ * flags. */
+#if defined(_MSC_VER) || defined(_WIN32)
+#define FLANG_LITTLE_ENDIAN 1
+#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#define FLANG_LITTLE_ENDIAN 1
+#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+#define FLANG_BIG_ENDIAN 1
+#else
+#error "Unknown or unsupported endianness."
+#endif
+
+#endif /* !defined(FLANG_LITTLE_ENDIAN) && !defined(FLANG_BIG_ENDIAN) */
+
 #endif /* !FORTRAN_RUNTIME_API_ATTRS_H_ */
diff --git a/flang/include/flang/Evaluate/common.h b/flang/include/flang/Evaluate/common.h
index 993540aebca57..fbfe411ebc977 100644
--- a/flang/include/flang/Evaluate/common.h
+++ b/flang/include/flang/Evaluate/common.h
@@ -9,6 +9,7 @@
 #ifndef FORTRAN_EVALUATE_COMMON_H_
 #define FORTRAN_EVALUATE_COMMON_H_
 
+#include "flang/Common/api-attrs.h"
 #include "flang/Common/enum-set.h"
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
diff --git a/flang/include/flang/Evaluate/traverse.h b/flang/include/flang/Evaluate/traverse.h
index dd38d64bff63f..9bb677e515372 100644
--- a/flang/include/flang/Evaluate/traverse.h
+++ b/flang/include/flang/Evaluate/traverse.h
@@ -159,9 +159,6 @@ class Traverse {
   template <typename T> Result operator()(const Designator<T> &x) const {
     return visitor_(x.u);
   }
-  template <typename T> Result operator()(const Variable<T> &x) const {
-    return visitor_(x.u);
-  }
   Result operator()(const DescriptorInquiry &x) const {
     return visitor_(x.base());
   }
diff --git a/flang/include/flang/Evaluate/variable.h b/flang/include/flang/Evaluate/variable.h
index 9b597d29813da..7f1518fd26e78 100644
--- a/flang/include/flang/Evaluate/variable.h
+++ b/flang/include/flang/Evaluate/variable.h
@@ -44,7 +44,6 @@ using SymbolVector = std::vector<SymbolRef>;
 
 // Forward declarations
 struct DataRef;
-template <typename T> struct Variable;
 
 // Reference a base object in memory.  This can be a Fortran symbol,
 // static data (e.g., CHARACTER literal), or compiler-created temporary.
diff --git a/flang/include/flang/Semantics/dump-expr.h b/flang/include/flang/Semantics/dump-expr.h
index 54c41300ecf36..2f445429a10b5 100644
--- a/flang/include/flang/Semantics/dump-expr.h
+++ b/flang/include/flang/Semantics/dump-expr.h
@@ -106,11 +106,6 @@ class DumpEvaluateExpr {
     Show(x.u);
     Outdent();
   }
-  template <typename T> void Show(const evaluate::Variable<T> &x) {
-    Indent("variable");
-    Show(x.u);
-    Outdent();
-  }
   void Show(const evaluate::DescriptorInquiry &x);
   void Show(const evaluate::SpecificIntrinsic &);
   void Show(const evaluate::ProcedureDesignator &x);
diff --git a/flang/lib/Lower/IterationSpace.cpp b/flang/lib/Lower/IterationSpace.cpp
index b011b3ab9a248..203fec508f795 100644
--- a/flang/lib/Lower/IterationSpace.cpp
+++ b/flang/lib/Lower/IterationSpace.cpp
@@ -165,10 +165,6 @@ class ArrayBaseFinder {
   RT find(const Fortran::evaluate::Designator<T> &x) {
     return find(x.u);
   }
-  template <typename T>
-  RT find(const Fortran::evaluate::Variable<T> &x) {
-    return find(x.u);
-  }
   RT find(const Fortran::evaluate::DescriptorInquiry &) { return {}; }
   RT find(const Fortran::evaluate::SpecificIntrinsic &) { return {}; }
   RT find(const Fortran::evaluate::ProcedureDesignator &x) { return {}; }
diff --git a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
index 0fbec8a204b8d..a01100511ec66 100644
--- a/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFOpConversion.cpp
@@ -810,7 +810,7 @@ struct CUFLaunchOpConversion
                   mlir::PatternRewriter &rewriter) const override {
     mlir::Location loc = op.getLoc();
     auto idxTy = mlir::IndexType::get(op.getContext());
-    auto zero = rewriter.create<mlir::arith::ConstantOp>(
+    mlir::Value zero = rewriter.create<mlir::arith::ConstantOp>(
         loc, rewriter.getIntegerType(32), rewriter.getI32IntegerAttr(0));
     auto gridSizeX =
         rewriter.create<mlir::arith::IndexCastOp>(loc, idxTy, op.getGridX());
@@ -869,10 +869,11 @@ struct CUFLaunchOpConversion
       }
       args.push_back(arg);
     }
-
+    mlir::Value dynamicShmemSize = op.getBytes() ? op.getBytes() : zero;
     auto gpuLaunchOp = rewriter.create<mlir::gpu::LaunchFuncOp>(
         loc, kernelName, mlir::gpu::KernelDim3{gridSizeX, gridSizeY, gridSizeZ},
-        mlir::gpu::KernelDim3{blockSizeX, blockSizeY, blockSizeZ}, zero, args);
+        mlir::gpu::KernelDim3{blockSizeX, blockSizeY, blockSizeZ},
+        dynamicShmemSize, args);
     if (clusterDimX && clusterDimY && clusterDimZ) {
       gpuLaunchOp.getClusterSizeXMutable().assign(clusterDimX);
       gpuLaunchOp.getClusterSizeYMutable().assign(clusterDimY);
diff --git a/flang/test/Fir/CUDA/cuda-launch.fir b/flang/test/Fir/CUDA/cuda-launch.fir
index 7833fc7b490bf..b8d79ca06ffd6 100644
--- a/flang/test/Fir/CUDA/cuda-launch.fir
+++ b/flang/test/Fir/CUDA/cuda-launch.fir
@@ -23,11 +23,15 @@ module attributes {gpu.container_module, dlti.dl_spec = #dlti.dl_spec<#dlti.dl_e
     // CHECK: %[[ALLOCA:.*]] = fir.alloca f32
     %c1 = arith.constant 1 : index
     %c11_i32 = arith.constant 11 : i32
+    %c1024_i32 = arith.constant 1024 : i32
     %c6_i32 = arith.constant 6 : i32
     %c1_i32 = arith.constant 1 : i32
     // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device1 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}}
     cuf.kernel_launch @cuda_device_mod::@_QPsub_device1<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32>>>()
 
+    // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device1 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c1024{{.*}}
+    cuf.kernel_launch @cuda_device_mod::@_QPsub_device1<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1024_i32>>>()
+
     // CHECK: gpu.launch_func  @cuda_device_mod::@_QPsub_device2 blocks in (%{{.*}}, %{{.*}}, %{{.*}}) threads in (%{{.*}}, %{{.*}}, %{{.*}})  dynamic_shared_memory_size %c0{{.*}} args(%[[ALLOCA]] : !fir.ref<f32>)
     cuf.kernel_launch @cuda_device_mod::@_QPsub_device2<<<%c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32, %c1_i32>>>(%0) : (!fir.ref<f32>)
     return
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index 4b03eaf4d02f2..d88bf844b15af 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -491,7 +491,7 @@ class FloatToString {
 
   LIBC_INLINE constexpr BlockInt get_negative_block(int block_index) {
     if (exponent < 0) {
-      const int32_t idx = -exponent / IDX_SIZE;
+      const int32_t idx = -exponent / static_cast<int32_t>(IDX_SIZE);
 
       UInt<MID_INT_SIZE> val;
 
@@ -579,7 +579,7 @@ class FloatToString {
 
     return num_requested_digits > -exponent;
 #else
-    const int32_t idx = -exponent / IDX_SIZE;
+    const int32_t idx = -exponent / static_cast<int32_t>(IDX_SIZE);
     const size_t p =
         POW10_OFFSET_2[idx] + negative_block_index - MIN_BLOCK_2[idx];
     // If the remaining digits are all 0, then this is the lowest block.
@@ -601,7 +601,7 @@ class FloatToString {
     }
     return 0;
 #else
-    return MIN_BLOCK_2[-exponent / IDX_SIZE];
+    return MIN_BLOCK_2[-exponent / static_cast<int32_t>(IDX_SIZE)];
 #endif
   }
 };
diff --git a/libc/test/UnitTest/ZxTest.h b/libc/test/UnitTest/ZxTest.h
index 0881902d62b3b..ac616dbcb03f0 100644
--- a/libc/test/UnitTest/ZxTest.h
+++ b/libc/test/UnitTest/ZxTest.h
@@ -14,14 +14,6 @@
 
 #define WITH_SIGNAL(X) #X
 
-// These macros are used in string unittests.
-#define ASSERT_ERRNO_EQ(VAL)                                                   \
-  ASSERT_EQ(VAL, static_cast<int>(LIBC_NAMESPACE::libc_errno))
-#define ASSERT_ERRNO_SUCCESS()                                                 \
-  ASSERT_EQ(0, static_cast<int>(LIBC_NAMESPACE::libc_errno))
-#define ASSERT_ERRNO_FAILURE()                                                 \
-  ASSERT_NE(0, static_cast<int>(LIBC_NAMESPACE::libc_errno))
-
 #ifndef EXPECT_DEATH
 // Since zxtest has ASSERT_DEATH but not EXPECT_DEATH, wrap calling it
 // in a lambda returning void to swallow any early returns so that this
diff --git a/libc/test/src/sys/epoll/linux/CMakeLists.txt b/libc/test/src/sys/epoll/linux/CMakeLists.txt
index 8f4b698ef1411..eba480c4b6f81 100644
--- a/libc/test/src/sys/epoll/linux/CMakeLists.txt
+++ b/libc/test/src/sys/epoll/linux/CMakeLists.txt
@@ -11,6 +11,7 @@ add_libc_unittest(
     libc.src.errno.errno
     libc.src.sys.epoll.epoll_create
     libc.src.unistd.close
+    libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -25,6 +26,7 @@ add_libc_unittest(
     libc.src.errno.errno
     libc.src.sys.epoll.epoll_create1
     libc.src.unistd.close
+    libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -42,6 +44,7 @@ add_libc_unittest(
     libc.src.sys.epoll.epoll_ctl
     libc.src.unistd.pipe
     libc.src.unistd.close
+    libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -60,6 +63,7 @@ add_libc_unittest(
     libc.src.sys.epoll.epoll_wait
     libc.src.unistd.pipe
     libc.src.unistd.close
+    libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -78,6 +82,7 @@ add_libc_unittest(
     libc.src.sys.epoll.epoll_pwait
     libc.src.unistd.pipe
     libc.src.unistd.close
+    libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -97,5 +102,6 @@ add_libc_unittest(
     libc.src.sys.epoll.epoll_pwait2
     libc.src.unistd.pipe
     libc.src.unistd.close
+    libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.ErrnoSetterMatcher
 )
diff --git a/libc/test/src/sys/epoll/linux/epoll_create1_test.cpp b/libc/test/src/sys/epoll/linux/epoll_create1_test.cpp
index 4059afe16b807..3fd62989308ac 100644
--- a/libc/test/src/sys/epoll/linux/epoll_create1_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_create1_test.cpp
@@ -6,15 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 #include "hdr/sys_epoll_macros.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/epoll/epoll_create1.h"
 #include "src/unistd/close.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+using LlvmLibcEpollCreate1Test = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
 
-TEST(LlvmLibcEpollCreate1Test, Basic) {
+TEST_F(LlvmLibcEpollCreate1Test, Basic) {
   int fd = LIBC_NAMESPACE::epoll_create1(0);
   ASSERT_GT(fd, 0);
   ASSERT_ERRNO_SUCCESS();
@@ -22,7 +23,7 @@ TEST(LlvmLibcEpollCreate1Test, Basic) {
   ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
 }
 
-TEST(LlvmLibcEpollCreate1Test, CloseOnExecute) {
+TEST_F(LlvmLibcEpollCreate1Test, CloseOnExecute) {
   int fd = LIBC_NAMESPACE::epoll_create1(EPOLL_CLOEXEC);
   ASSERT_GT(fd, 0);
   ASSERT_ERRNO_SUCCESS();
diff --git a/libc/test/src/sys/epoll/linux/epoll_create_test.cpp b/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
index 9c4bad10c8384..06c17c6cf29e6 100644
--- a/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
@@ -5,16 +5,17 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#include "src/errno/libc_errno.h"
 #include "src/sys/epoll/epoll_create.h"
 #include "src/unistd/close.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 #include <sys/syscall.h> // For syscall numbers.
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+using LlvmLibcEpollCreateTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
 
-TEST(LlvmLibcEpollCreateTest, Basic) {
+TEST_F(LlvmLibcEpollCreateTest, Basic) {
   int fd = LIBC_NAMESPACE::epoll_create(1);
   ASSERT_GT(fd, 0);
   ASSERT_ERRNO_SUCCESS();
@@ -23,7 +24,7 @@ TEST(LlvmLibcEpollCreateTest, Basic) {
 }
 
 #ifdef SYS_epoll_create
-TEST(LlvmLibcEpollCreateTest, Fails) {
+TEST_F(LlvmLibcEpollCreateTest, Fails) {
   ASSERT_THAT(LIBC_NAMESPACE::epoll_create(0), Fails(EINVAL));
 }
 #endif
diff --git a/libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp b/libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp
index fa2d358c57966..bfbf9c09f7bed 100644
--- a/libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp
@@ -8,17 +8,18 @@
 
 #include "hdr/sys_epoll_macros.h"
 #include "hdr/types/struct_epoll_event.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/epoll/epoll_create1.h"
 #include "src/sys/epoll/epoll_ctl.h"
 #include "src/unistd/close.h"
 #include "src/unistd/pipe.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+using LlvmLibcEpollCtlTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
 
-TEST(LlvmLibcEpollCtlTest, Basic) {
+TEST_F(LlvmLibcEpollCtlTest, Basic) {
   int epfd = LIBC_NAMESPACE::epoll_create1(0);
   ASSERT_GT(epfd, 0);
   ASSERT_ERRNO_SUCCESS();
diff --git a/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp b/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp
index 2f4c9854be1a0..6da070e2561e8 100644
--- a/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp
@@ -8,18 +8,19 @@
 #include "hdr/sys_epoll_macros.h"
 #include "hdr/types/struct_epoll_event.h"
 #include "hdr/types/struct_timespec.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/epoll/epoll_create1.h"
 #include "src/sys/epoll/epoll_ctl.h"
 #include "src/sys/epoll/epoll_pwait2.h"
 #include "src/unistd/close.h"
 #include "src/unistd/pipe.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+using LlvmLibcEpollPwaitTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
 
-TEST(LlvmLibcEpollPwaitTest, Basic) {
+TEST_F(LlvmLibcEpollPwaitTest, Basic) {
   int epfd = LIBC_NAMESPACE::epoll_create1(0);
   ASSERT_GT(epfd, 0);
   ASSERT_ERRNO_SUCCESS();
diff --git a/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp b/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp
index 8e14aea8b9d57..3b936177ff801 100644
--- a/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp
@@ -7,19 +7,19 @@
 //===----------------------------------------------------------------------===//
 #include "hdr/sys_epoll_macros.h"
 #include "hdr/types/struct_epoll_event.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/epoll/epoll_create1.h"
 #include "src/sys/epoll/epoll_ctl.h"
 #include "src/sys/epoll/epoll_pwait.h"
 #include "src/unistd/close.h"
 #include "src/unistd/pipe.h"
-
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+using LlvmLibcEpollPwaitTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
 
-TEST(LlvmLibcEpollPwaitTest, Basic) {
+TEST_F(LlvmLibcEpollPwaitTest, Basic) {
   int epfd = LIBC_NAMESPACE::epoll_create1(0);
   ASSERT_GT(epfd, 0);
   ASSERT_ERRNO_SUCCESS();
diff --git a/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp b/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp
index f9e855a891b05..7457ef22e6747 100644
--- a/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp
@@ -7,18 +7,19 @@
 //===----------------------------------------------------------------------===//
 #include "hdr/sys_epoll_macros.h"
 #include "hdr/types/struct_epoll_event.h"
-#include "src/errno/libc_errno.h"
 #include "src/sys/epoll/epoll_create1.h"
 #include "src/sys/epoll/epoll_ctl.h"
 #include "src/sys/epoll/epoll_wait.h"
 #include "src/unistd/close.h"
 #include "src/unistd/pipe.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+using LlvmLibcEpollWaitTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
 
-TEST(LlvmLibcEpollWaitTest, Basic) {
+TEST_F(LlvmLibcEpollWaitTest, Basic) {
   int epfd = LIBC_NAMESPACE::epoll_create1(0);
   ASSERT_GT(epfd, 0);
   ASSERT_ERRNO_SUCCESS();
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 1e39f761ecdad..774d86afb44fa 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -147,6 +147,9 @@ Changes to the RISC-V Backend
 * Adds assembler support for the 'Zclsd` (Compressed Load/Store Pair Instructions)
   extension.
 * Adds experimental assembler support for Zvqdotq.
+* Adds Support for Qualcomm's `qci-nest` and `qci-nonest` interrupt types, which
+  use instructions from `Xqciint` to save and restore some GPRs during interrupt
+  handlers.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index bd5a796c0b31c..28905e27837a7 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -320,7 +320,7 @@ ArrayRef<EnumEntry<ResourceKind>> getResourceKinds();
 
 #define RESOURCE_FLAG(Index, Enum) bool Enum = false;
 struct ResourceFlags {
-  ResourceFlags() {};
+  ResourceFlags() : Flags(0U) {};
   struct FlagsBits {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
   };
diff --git a/llvm/include/llvm/Support/Mustache.h b/llvm/include/llvm/Support/Mustache.h
new file mode 100644
index 0000000000000..9cd96739f2227
--- /dev/null
+++ b/llvm/include/llvm/Support/Mustache.h
@@ -0,0 +1,128 @@
+//===--- Mustache.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of the Mustache templating language supports version 1.4.2
+// currently relies on llvm::json::Value for data input.
+// See the Mustache spec for more information
+// (https://mustache.github.io/mustache.5.html).
+//
+// Current Features Supported:
+// - Variables
+// - Sections
+// - Inverted Sections
+// - Partials
+// - Comments
+// - Lambdas
+// - Unescaped Variables
+//
+// Features Not Supported:
+// - Set Delimiter
+// - Blocks
+// - Parents
+// - Dynamic Names
+//
+// The Template class is a container class that outputs the Mustache template
+// string and is the main class for users. It stores all the lambdas and the
+// ASTNode Tree. When the Template is instantiated it tokenizes the Template
+// String and creates a vector of Tokens. Then it calls a basic recursive
+// descent parser to construct the ASTNode Tree. The ASTNodes are all stored
+// in an arena allocator which is freed once the template class goes out of
+// scope.
+//
+// Usage:
+// \code
+//   // Creating a simple template and rendering it
+//   auto Template = Template("Hello, {{name}}!");
+//   Value Data = {{"name", "World"}};
+//   std::string Out;
+//   raw_string_ostream OS(Out);
+//   T.render(Data, OS);
+//   // Out == "Hello, World!"
+//
+//   // Creating a template with a partial and rendering it
+//   auto Template = Template("{{>partial}}");
+//   Template.registerPartial("partial", "Hello, {{name}}!");
+//   Value Data = {{"name", "World"}};
+//   std::string Out;
+//   raw_string_ostream OS(Out);
+//   T.render(Data, OS);
+//   // Out == "Hello, World!"
+//
+//   // Creating a template with a lambda and rendering it
+//   Value D = Object{};
+//   auto T = Template("Hello, {{lambda}}!");
+//   Lambda L = []() -> llvm::json::Value { return "World"; };
+//   T.registerLambda("lambda", L);
+//   std::string Out;
+//   raw_string_ostream OS(Out);
+//   T.render(D, OS);
+//   // Out == "Hello, World!"
+// \endcode
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_MUSTACHE
+#define LLVM_SUPPORT_MUSTACHE
+
+#include "Error.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/StringSaver.h"
+#include <functional>
+#include <vector>
+
+namespace llvm::mustache {
+
+using Lambda = std::function<llvm::json::Value()>;
+using SectionLambda = std::function<llvm::json::Value(std::string)>;
+
+class ASTNode;
+using AstPtr = std::unique_ptr<ASTNode>;
+
+// A Template represents the container for the AST and the partials
+// and Lambdas that are registered with it.
+class Template {
+public:
+  Template(StringRef TemplateStr);
+
+  Template(const Template &) = delete;
+
+  Template &operator=(const Template &) = delete;
+
+  Template(Template &&Other) noexcept;
+
+  // Define this in the cpp file to  work around ASTNode being an incomplete
+  // type.
+  ~Template();
+
+  Template &operator=(Template &&Other) noexcept;
+
+  void render(const llvm::json::Value &Data, llvm::raw_ostream &OS);
+
+  void registerPartial(std::string Name, std::string Partial);
+
+  void registerLambda(std::string Name, Lambda Lambda);
+
+  void registerLambda(std::string Name, SectionLambda Lambda);
+
+  // By default the Mustache Spec Specifies that HTML special characters
+  // should be escaped. This function allows the user to specify which
+  // characters should be escaped.
+  void overrideEscapeCharacters(DenseMap<char, std::string> Escapes);
+
+private:
+  StringMap<AstPtr> Partials;
+  StringMap<Lambda> Lambdas;
+  StringMap<SectionLambda> SectionLambdas;
+  DenseMap<char, std::string> Escapes;
+  AstPtr Tree;
+};
+} // namespace llvm::mustache
+
+#endif // LLVM_SUPPORT_MUSTACHE
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 49a26a618de83..2754c97fce6c1 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -220,6 +220,7 @@ add_llvm_component_library(LLVMSupport
   MD5.cpp
   MSP430Attributes.cpp
   MSP430AttributeParser.cpp
+  Mustache.cpp      
   NativeFormatting.cpp
   OptimizedStructLayout.cpp
   Optional.cpp
diff --git a/llvm/lib/Support/Mustache.cpp b/llvm/lib/Support/Mustache.cpp
new file mode 100644
index 0000000000000..2560619538f9a
--- /dev/null
+++ b/llvm/lib/Support/Mustache.cpp
@@ -0,0 +1,763 @@
+//===-- Mustache.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/Support/Mustache.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include <sstream>
+
+using namespace llvm;
+using namespace llvm::mustache;
+
+namespace {
+
+using Accessor = SmallVector<std::string>;
+
+static bool isFalsey(const json::Value &V) {
+  return V.getAsNull() || (V.getAsBoolean() && !V.getAsBoolean().value()) ||
+         (V.getAsArray() && V.getAsArray()->empty());
+}
+
+static Accessor splitMustacheString(StringRef Str) {
+  // We split the mustache string into an accessor.
+  // For example:
+  //    "a.b.c" would be split into {"a", "b", "c"}
+  // We make an exception for a single dot which
+  // refers to the current context.
+  Accessor Tokens;
+  if (Str == ".") {
+    Tokens.emplace_back(Str);
+    return Tokens;
+  }
+  while (!Str.empty()) {
+    StringRef Part;
+    std::tie(Part, Str) = Str.split(".");
+    Tokens.emplace_back(Part.trim());
+  }
+  return Tokens;
+}
+} // namespace
+
+namespace llvm::mustache {
+
+class Token {
+public:
+  enum class Type {
+    Text,
+    Variable,
+    Partial,
+    SectionOpen,
+    SectionClose,
+    InvertSectionOpen,
+    UnescapeVariable,
+    Comment,
+  };
+
+  Token(std::string Str)
+      : TokenType(Type::Text), RawBody(std::move(Str)), TokenBody(RawBody),
+        AccessorValue({}), Indentation(0) {};
+
+  Token(std::string RawBody, std::string TokenBody, char Identifier)
+      : RawBody(std::move(RawBody)), TokenBody(std::move(TokenBody)),
+        Indentation(0) {
+    TokenType = getTokenType(Identifier);
+    if (TokenType == Type::Comment)
+      return;
+    StringRef AccessorStr(this->TokenBody);
+    if (TokenType != Type::Variable)
+      AccessorStr = AccessorStr.substr(1);
+    AccessorValue = splitMustacheString(StringRef(AccessorStr).trim());
+  }
+
+  Accessor getAccessor() const { return AccessorValue; }
+
+  Type getType() const { return TokenType; }
+
+  void setIndentation(size_t NewIndentation) { Indentation = NewIndentation; }
+
+  size_t getIndentation() const { return Indentation; }
+
+  static Type getTokenType(char Identifier) {
+    switch (Identifier) {
+    case '#':
+      return Type::SectionOpen;
+    case '/':
+      return Type::SectionClose;
+    case '^':
+      return Type::InvertSectionOpen;
+    case '!':
+      return Type::Comment;
+    case '>':
+      return Type::Partial;
+    case '&':
+      return Type::UnescapeVariable;
+    default:
+      return Type::Variable;
+    }
+  }
+
+  Type TokenType;
+  // RawBody is the original string that was tokenized.
+  std::string RawBody;
+  // TokenBody is the original string with the identifier removed.
+  std::string TokenBody;
+  Accessor AccessorValue;
+  size_t Indentation;
+};
+
+class ASTNode {
+public:
+  enum Type {
+    Root,
+    Text,
+    Partial,
+    Variable,
+    UnescapeVariable,
+    Section,
+    InvertSection,
+  };
+
+  ASTNode(llvm::StringMap<AstPtr> &Partials, llvm::StringMap<Lambda> &Lambdas,
+          llvm::StringMap<SectionLambda> &SectionLambdas,
+          llvm::DenseMap<char, std::string> &Escapes)
+      : Partials(Partials), Lambdas(Lambdas), SectionLambdas(SectionLambdas),
+        Escapes(Escapes), Ty(Type::Root), Parent(nullptr),
+        ParentContext(nullptr) {}
+
+  ASTNode(std::string Body, ASTNode *Parent, llvm::StringMap<AstPtr> &Partials,
+          llvm::StringMap<Lambda> &Lambdas,
+          llvm::StringMap<SectionLambda> &SectionLambdas,
+          llvm::DenseMap<char, std::string> &Escapes)
+      : Partials(Partials), Lambdas(Lambdas), SectionLambdas(SectionLambdas),
+        Escapes(Escapes), Ty(Type::Text), Body(std::move(Body)), Parent(Parent),
+        ParentContext(nullptr) {}
+
+  // Constructor for Section/InvertSection/Variable/UnescapeVariable Nodes
+  ASTNode(Type Ty, Accessor Accessor, ASTNode *Parent,
+          llvm::StringMap<AstPtr> &Partials, llvm::StringMap<Lambda> &Lambdas,
+          llvm::StringMap<SectionLambda> &SectionLambdas,
+          llvm::DenseMap<char, std::string> &Escapes)
+      : Partials(Partials), Lambdas(Lambdas), SectionLambdas(SectionLambdas),
+        Escapes(Escapes), Ty(Ty), Parent(Parent),
+        AccessorValue(std::move(Accessor)), ParentContext(nullptr) {}
+
+  void addChild(AstPtr Child) { Children.emplace_back(std::move(Child)); };
+
+  void setRawBody(std::string NewBody) { RawBody = std::move(NewBody); };
+
+  void setIndentation(size_t NewIndentation) { Indentation = NewIndentation; };
+
+  void render(const llvm::json::Value &Data, llvm::raw_ostream &OS);
+
+private:
+  void renderLambdas(const llvm::json::Value &Contexts, llvm::raw_ostream &OS,
+                     Lambda &L);
+
+  void renderSectionLambdas(const llvm::json::Value &Contexts,
+                            llvm::raw_ostream &OS, SectionLambda &L);
+
+  void renderPartial(const llvm::json::Value &Contexts, llvm::raw_ostream &OS,
+                     ASTNode *Partial);
+
+  void renderChild(const llvm::json::Value &Context, llvm::raw_ostream &OS);
+
+  const llvm::json::Value *findContext();
+
+  StringMap<AstPtr> &Partials;
+  StringMap<Lambda> &Lambdas;
+  StringMap<SectionLambda> &SectionLambdas;
+  DenseMap<char, std::string> &Escapes;
+  Type Ty;
+  size_t Indentation = 0;
+  std::string RawBody;
+  std::string Body;
+  ASTNode *Parent;
+  // TODO: switch implementation to SmallVector<T>
+  std::vector<AstPtr> Children;
+  const Accessor AccessorValue;
+  const llvm::json::Value *ParentContext;
+};
+
+// A wrapper for arena allocator for ASTNodes
+AstPtr createRootNode(llvm::StringMap<AstPtr> &Partials,
+                      llvm::StringMap<Lambda> &Lambdas,
+                      llvm::StringMap<SectionLambda> &SectionLambdas,
+                      llvm::DenseMap<char, std::string> &Escapes) {
+  return std::make_unique<ASTNode>(Partials, Lambdas, SectionLambdas, Escapes);
+}
+
+AstPtr createNode(ASTNode::Type T, Accessor A, ASTNode *Parent,
+                  llvm::StringMap<AstPtr> &Partials,
+                  llvm::StringMap<Lambda> &Lambdas,
+                  llvm::StringMap<SectionLambda> &SectionLambdas,
+                  llvm::DenseMap<char, std::string> &Escapes) {
+  return std::make_unique<ASTNode>(T, std::move(A), Parent, Partials, Lambdas,
+                                   SectionLambdas, Escapes);
+}
+
+AstPtr createTextNode(std::string Body, ASTNode *Parent,
+                      llvm::StringMap<AstPtr> &Partials,
+                      llvm::StringMap<Lambda> &Lambdas,
+                      llvm::StringMap<SectionLambda> &SectionLambdas,
+                      llvm::DenseMap<char, std::string> &Escapes) {
+  return std::make_unique<ASTNode>(std::move(Body), Parent, Partials, Lambdas,
+                                   SectionLambdas, Escapes);
+}
+
+// Function to check if there is meaningful text behind.
+// We determine if a token has meaningful text behind
+// if the right of previous token contains anything that is
+// not a newline.
+// For example:
+//  "Stuff {{#Section}}" (returns true)
+//   vs
+//  "{{#Section}} \n" (returns false)
+// We make an exception for when previous token is empty
+// and the current token is the second token.
+// For example:
+//  "{{#Section}}"
+bool hasTextBehind(size_t Idx, const ArrayRef<Token> &Tokens) {
+  if (Idx == 0)
+    return true;
+
+  size_t PrevIdx = Idx - 1;
+  if (Tokens[PrevIdx].getType() != Token::Type::Text)
+    return true;
+
+  const Token &PrevToken = Tokens[PrevIdx];
+  StringRef TokenBody = StringRef(PrevToken.RawBody).rtrim(" \r\t\v");
+  return !TokenBody.ends_with("\n") && !(TokenBody.empty() && Idx == 1);
+}
+
+// Function to check if there's no meaningful text ahead.
+// We determine if a token has text ahead if the left of previous
+// token does not start with a newline.
+bool hasTextAhead(size_t Idx, const ArrayRef<Token> &Tokens) {
+  if (Idx >= Tokens.size() - 1)
+    return true;
+
+  size_t NextIdx = Idx + 1;
+  if (Tokens[NextIdx].getType() != Token::Type::Text)
+    return true;
+
+  const Token &NextToken = Tokens[NextIdx];
+  StringRef TokenBody = StringRef(NextToken.RawBody).ltrim(" ");
+  return !TokenBody.starts_with("\r\n") && !TokenBody.starts_with("\n");
+}
+
+bool requiresCleanUp(Token::Type T) {
+  // We must clean up all the tokens that could contain child nodes.
+  return T == Token::Type::SectionOpen || T == Token::Type::InvertSectionOpen ||
+         T == Token::Type::SectionClose || T == Token::Type::Comment ||
+         T == Token::Type::Partial;
+}
+
+// Adjust next token body if there is no text ahead.
+// For example:
+// The template string
+//  "{{! Comment }} \nLine 2"
+// would be considered as no text ahead and should be rendered as
+//  " Line 2"
+void stripTokenAhead(SmallVectorImpl<Token> &Tokens, size_t Idx) {
+  Token &NextToken = Tokens[Idx + 1];
+  StringRef NextTokenBody = NextToken.TokenBody;
+  // Cut off the leading newline which could be \n or \r\n.
+  if (NextTokenBody.starts_with("\r\n"))
+    NextToken.TokenBody = NextTokenBody.substr(2).str();
+  else if (NextTokenBody.starts_with("\n"))
+    NextToken.TokenBody = NextTokenBody.substr(1).str();
+}
+
+// Adjust previous token body if there no text behind.
+// For example:
+//  The template string
+//  " \t{{#section}}A{{/section}}"
+// would be considered as having no text ahead and would be render as
+//  "A"
+// The exception for this is partial tag which requires us to
+// keep track of the indentation once it's rendered.
+void stripTokenBefore(SmallVectorImpl<Token> &Tokens, size_t Idx,
+                      Token &CurrentToken, Token::Type CurrentType) {
+  Token &PrevToken = Tokens[Idx - 1];
+  StringRef PrevTokenBody = PrevToken.TokenBody;
+  StringRef Unindented = PrevTokenBody.rtrim(" \r\t\v");
+  size_t Indentation = PrevTokenBody.size() - Unindented.size();
+  if (CurrentType != Token::Type::Partial)
+    PrevToken.TokenBody = Unindented.str();
+  CurrentToken.setIndentation(Indentation);
+}
+
+// Simple tokenizer that splits the template into tokens.
+// The mustache spec allows {{{ }}} to unescape variables,
+// but we don't support that here. An unescape variable
+// is represented only by {{& variable}}.
+SmallVector<Token> tokenize(StringRef Template) {
+  SmallVector<Token> Tokens;
+  StringLiteral Open("{{");
+  StringLiteral Close("}}");
+  size_t Start = 0;
+  size_t DelimiterStart = Template.find(Open);
+  if (DelimiterStart == StringRef::npos) {
+    Tokens.emplace_back(Template.str());
+    return Tokens;
+  }
+  while (DelimiterStart != StringRef::npos) {
+    if (DelimiterStart != Start)
+      Tokens.emplace_back(Template.substr(Start, DelimiterStart - Start).str());
+    size_t DelimiterEnd = Template.find(Close, DelimiterStart);
+    if (DelimiterEnd == StringRef::npos)
+      break;
+
+    // Extract the Interpolated variable without delimiters.
+    size_t InterpolatedStart = DelimiterStart + Open.size();
+    size_t InterpolatedEnd = DelimiterEnd - DelimiterStart - Close.size();
+    std::string Interpolated =
+        Template.substr(InterpolatedStart, InterpolatedEnd).str();
+    std::string RawBody = Open.str() + Interpolated + Close.str();
+    Tokens.emplace_back(RawBody, Interpolated, Interpolated[0]);
+    Start = DelimiterEnd + Close.size();
+    DelimiterStart = Template.find(Open, Start);
+  }
+
+  if (Start < Template.size())
+    Tokens.emplace_back(Template.substr(Start).str());
+
+  // Fix up white spaces for:
+  //   - open sections
+  //   - inverted sections
+  //   - close sections
+  //   - comments
+  //
+  // This loop attempts to find standalone tokens and tries to trim out
+  // the surrounding whitespace.
+  // For example:
+  // if you have the template string
+  //  {{#section}} \n Example \n{{/section}}
+  // The output should would be
+  // For example:
+  //  \n Example \n
+  size_t LastIdx = Tokens.size() - 1;
+  for (size_t Idx = 0, End = Tokens.size(); Idx < End; ++Idx) {
+    Token &CurrentToken = Tokens[Idx];
+    Token::Type CurrentType = CurrentToken.getType();
+    // Check if token type requires cleanup.
+    bool RequiresCleanUp = requiresCleanUp(CurrentType);
+
+    if (!RequiresCleanUp)
+      continue;
+
+    // We adjust the token body if there's no text behind or ahead.
+    // A token is considered to have no text ahead if the right of the previous
+    // token is a newline followed by spaces.
+    // A token is considered to have no text behind if the left of the next
+    // token is spaces followed by a newline.
+    // eg.
+    //  "Line 1\n {{#section}} \n Line 2 \n {{/section}} \n Line 3"
+    bool HasTextBehind = hasTextBehind(Idx, Tokens);
+    bool HasTextAhead = hasTextAhead(Idx, Tokens);
+
+    if ((!HasTextAhead && !HasTextBehind) || (!HasTextAhead && Idx == 0))
+      stripTokenAhead(Tokens, Idx);
+
+    if ((!HasTextBehind && !HasTextAhead) || (!HasTextBehind && Idx == LastIdx))
+      stripTokenBefore(Tokens, Idx, CurrentToken, CurrentType);
+  }
+  return Tokens;
+}
+
+// Custom stream to escape strings.
+class EscapeStringStream : public raw_ostream {
+public:
+  explicit EscapeStringStream(llvm::raw_ostream &WrappedStream,
+                              DenseMap<char, std::string> &Escape)
+      : Escape(Escape), WrappedStream(WrappedStream) {
+    SetUnbuffered();
+  }
+
+protected:
+  void write_impl(const char *Ptr, size_t Size) override {
+    llvm::StringRef Data(Ptr, Size);
+    for (char C : Data) {
+      auto It = Escape.find(C);
+      if (It != Escape.end())
+        WrappedStream << It->getSecond();
+      else
+        WrappedStream << C;
+    }
+  }
+
+  uint64_t current_pos() const override { return WrappedStream.tell(); }
+
+private:
+  DenseMap<char, std::string> &Escape;
+  llvm::raw_ostream &WrappedStream;
+};
+
+// Custom stream to add indentation used to for rendering partials.
+class AddIndentationStringStream : public raw_ostream {
+public:
+  explicit AddIndentationStringStream(llvm::raw_ostream &WrappedStream,
+                                      size_t Indentation)
+      : Indentation(Indentation), WrappedStream(WrappedStream) {
+    SetUnbuffered();
+  }
+
+protected:
+  void write_impl(const char *Ptr, size_t Size) override {
+    llvm::StringRef Data(Ptr, Size);
+    SmallString<0> Indent;
+    Indent.resize(Indentation, ' ');
+    for (char C : Data) {
+      WrappedStream << C;
+      if (C == '\n')
+        WrappedStream << Indent;
+    }
+  }
+
+  uint64_t current_pos() const override { return WrappedStream.tell(); }
+
+private:
+  size_t Indentation;
+  llvm::raw_ostream &WrappedStream;
+};
+
+class Parser {
+public:
+  Parser(StringRef TemplateStr) : TemplateStr(TemplateStr) {}
+
+  AstPtr parse(llvm::StringMap<AstPtr> &Partials,
+               llvm::StringMap<Lambda> &Lambdas,
+               llvm::StringMap<SectionLambda> &SectionLambdas,
+               llvm::DenseMap<char, std::string> &Escapes);
+
+private:
+  void parseMustache(ASTNode *Parent, llvm::StringMap<AstPtr> &Partials,
+                     llvm::StringMap<Lambda> &Lambdas,
+                     llvm::StringMap<SectionLambda> &SectionLambdas,
+                     llvm::DenseMap<char, std::string> &Escapes);
+
+  SmallVector<Token> Tokens;
+  size_t CurrentPtr;
+  StringRef TemplateStr;
+};
+
+AstPtr Parser::parse(llvm::StringMap<AstPtr> &Partials,
+                     llvm::StringMap<Lambda> &Lambdas,
+                     llvm::StringMap<SectionLambda> &SectionLambdas,
+                     llvm::DenseMap<char, std::string> &Escapes) {
+  Tokens = tokenize(TemplateStr);
+  CurrentPtr = 0;
+  AstPtr RootNode = createRootNode(Partials, Lambdas, SectionLambdas, Escapes);
+  parseMustache(RootNode.get(), Partials, Lambdas, SectionLambdas, Escapes);
+  return RootNode;
+}
+
+void Parser::parseMustache(ASTNode *Parent, llvm::StringMap<AstPtr> &Partials,
+                           llvm::StringMap<Lambda> &Lambdas,
+                           llvm::StringMap<SectionLambda> &SectionLambdas,
+                           llvm::DenseMap<char, std::string> &Escapes) {
+
+  while (CurrentPtr < Tokens.size()) {
+    Token CurrentToken = Tokens[CurrentPtr];
+    CurrentPtr++;
+    Accessor A = CurrentToken.getAccessor();
+    AstPtr CurrentNode;
+
+    switch (CurrentToken.getType()) {
+    case Token::Type::Text: {
+      CurrentNode = createTextNode(std::move(CurrentToken.TokenBody), Parent,
+                                   Partials, Lambdas, SectionLambdas, Escapes);
+      Parent->addChild(std::move(CurrentNode));
+      break;
+    }
+    case Token::Type::Variable: {
+      CurrentNode = createNode(ASTNode::Variable, std::move(A), Parent,
+                               Partials, Lambdas, SectionLambdas, Escapes);
+      Parent->addChild(std::move(CurrentNode));
+      break;
+    }
+    case Token::Type::UnescapeVariable: {
+      CurrentNode = createNode(ASTNode::UnescapeVariable, std::move(A), Parent,
+                               Partials, Lambdas, SectionLambdas, Escapes);
+      Parent->addChild(std::move(CurrentNode));
+      break;
+    }
+    case Token::Type::Partial: {
+      CurrentNode = createNode(ASTNode::Partial, std::move(A), Parent, Partials,
+                               Lambdas, SectionLambdas, Escapes);
+      CurrentNode->setIndentation(CurrentToken.getIndentation());
+      Parent->addChild(std::move(CurrentNode));
+      break;
+    }
+    case Token::Type::SectionOpen: {
+      CurrentNode = createNode(ASTNode::Section, A, Parent, Partials, Lambdas,
+                               SectionLambdas, Escapes);
+      size_t Start = CurrentPtr;
+      parseMustache(CurrentNode.get(), Partials, Lambdas, SectionLambdas,
+                    Escapes);
+      const size_t End = CurrentPtr - 1;
+      std::string RawBody;
+      for (std::size_t I = Start; I < End; I++)
+        RawBody += Tokens[I].RawBody;
+      CurrentNode->setRawBody(std::move(RawBody));
+      Parent->addChild(std::move(CurrentNode));
+      break;
+    }
+    case Token::Type::InvertSectionOpen: {
+      CurrentNode = createNode(ASTNode::InvertSection, A, Parent, Partials,
+                               Lambdas, SectionLambdas, Escapes);
+      size_t Start = CurrentPtr;
+      parseMustache(CurrentNode.get(), Partials, Lambdas, SectionLambdas,
+                    Escapes);
+      const size_t End = CurrentPtr - 1;
+      std::string RawBody;
+      for (size_t Idx = Start; Idx < End; Idx++)
+        RawBody += Tokens[Idx].RawBody;
+      CurrentNode->setRawBody(std::move(RawBody));
+      Parent->addChild(std::move(CurrentNode));
+      break;
+    }
+    case Token::Type::Comment:
+      break;
+    case Token::Type::SectionClose:
+      return;
+    }
+  }
+}
+void toMustacheString(const json::Value &Data, raw_ostream &OS) {
+  switch (Data.kind()) {
+  case json::Value::Null:
+    return;
+  case json::Value::Number: {
+    auto Num = *Data.getAsNumber();
+    std::ostringstream SS;
+    SS << Num;
+    OS << SS.str();
+    return;
+  }
+  case json::Value::String: {
+    auto Str = *Data.getAsString();
+    OS << Str.str();
+    return;
+  }
+
+  case json::Value::Array: {
+    auto Arr = *Data.getAsArray();
+    if (Arr.empty())
+      return;
+    [[fallthrough]];
+  }
+  case json::Value::Object:
+  case json::Value::Boolean: {
+    llvm::json::OStream JOS(OS, 2);
+    JOS.value(Data);
+    break;
+  }
+  }
+}
+
+void ASTNode::render(const json::Value &Data, raw_ostream &OS) {
+  ParentContext = &Data;
+  const json::Value *ContextPtr = Ty == Root ? ParentContext : findContext();
+  const json::Value &Context = ContextPtr ? *ContextPtr : nullptr;
+
+  switch (Ty) {
+  case Root:
+    renderChild(Data, OS);
+    return;
+  case Text:
+    OS << Body;
+    return;
+  case Partial: {
+    auto Partial = Partials.find(AccessorValue[0]);
+    if (Partial != Partials.end())
+      renderPartial(Data, OS, Partial->getValue().get());
+    return;
+  }
+  case Variable: {
+    auto Lambda = Lambdas.find(AccessorValue[0]);
+    if (Lambda != Lambdas.end())
+      renderLambdas(Data, OS, Lambda->getValue());
+    else {
+      EscapeStringStream ES(OS, Escapes);
+      toMustacheString(Context, ES);
+    }
+    return;
+  }
+  case UnescapeVariable: {
+    auto Lambda = Lambdas.find(AccessorValue[0]);
+    if (Lambda != Lambdas.end())
+      renderLambdas(Data, OS, Lambda->getValue());
+    else
+      toMustacheString(Context, OS);
+    return;
+  }
+  case Section: {
+    // Sections are not rendered if the context is falsey.
+    auto SectionLambda = SectionLambdas.find(AccessorValue[0]);
+    bool IsLambda = SectionLambda != SectionLambdas.end();
+    if (isFalsey(Context) && !IsLambda)
+      return;
+
+    if (IsLambda) {
+      renderSectionLambdas(Data, OS, SectionLambda->getValue());
+      return;
+    }
+
+    if (Context.getAsArray()) {
+      const json::Array *Arr = Context.getAsArray();
+      for (const json::Value &V : *Arr)
+        renderChild(V, OS);
+      return;
+    }
+    renderChild(Context, OS);
+    return;
+  }
+  case InvertSection: {
+    bool IsLambda =
+        SectionLambdas.find(AccessorValue[0]) != SectionLambdas.end();
+    if (!isFalsey(Context) || IsLambda)
+      return;
+    renderChild(Context, OS);
+    return;
+  }
+  }
+  llvm_unreachable("Invalid ASTNode type");
+}
+
+const json::Value *ASTNode::findContext() {
+  // The mustache spec allows for dot notation to access nested values
+  // a single dot refers to the current context.
+  // We attempt to find the JSON context in the current node, if it is not
+  // found, then we traverse the parent nodes to find the context until we
+  // reach the root node or the context is found.
+  if (AccessorValue.empty())
+    return nullptr;
+  if (AccessorValue[0] == ".")
+    return ParentContext;
+
+  const json::Object *CurrentContext = ParentContext->getAsObject();
+  StringRef CurrentAccessor = AccessorValue[0];
+  ASTNode *CurrentParent = Parent;
+
+  while (!CurrentContext || !CurrentContext->get(CurrentAccessor)) {
+    if (CurrentParent->Ty != Root) {
+      CurrentContext = CurrentParent->ParentContext->getAsObject();
+      CurrentParent = CurrentParent->Parent;
+      continue;
+    }
+    return nullptr;
+  }
+  const json::Value *Context = nullptr;
+  for (auto [Idx, Acc] : enumerate(AccessorValue)) {
+    const json::Value *CurrentValue = CurrentContext->get(Acc);
+    if (!CurrentValue)
+      return nullptr;
+    if (Idx < AccessorValue.size() - 1) {
+      CurrentContext = CurrentValue->getAsObject();
+      if (!CurrentContext)
+        return nullptr;
+    } else
+      Context = CurrentValue;
+  }
+  return Context;
+}
+
+void ASTNode::renderChild(const json::Value &Contexts, llvm::raw_ostream &OS) {
+  for (AstPtr &Child : Children)
+    Child->render(Contexts, OS);
+}
+
+void ASTNode::renderPartial(const json::Value &Contexts, llvm::raw_ostream &OS,
+                            ASTNode *Partial) {
+  AddIndentationStringStream IS(OS, Indentation);
+  Partial->render(Contexts, IS);
+}
+
+void ASTNode::renderLambdas(const json::Value &Contexts, llvm::raw_ostream &OS,
+                            Lambda &L) {
+  json::Value LambdaResult = L();
+  std::string LambdaStr;
+  raw_string_ostream Output(LambdaStr);
+  toMustacheString(LambdaResult, Output);
+  Parser P = Parser(LambdaStr);
+  AstPtr LambdaNode = P.parse(Partials, Lambdas, SectionLambdas, Escapes);
+
+  EscapeStringStream ES(OS, Escapes);
+  if (Ty == Variable) {
+    LambdaNode->render(Contexts, ES);
+    return;
+  }
+  LambdaNode->render(Contexts, OS);
+}
+
+void ASTNode::renderSectionLambdas(const json::Value &Contexts,
+                                   llvm::raw_ostream &OS, SectionLambda &L) {
+  json::Value Return = L(RawBody);
+  if (isFalsey(Return))
+    return;
+  std::string LambdaStr;
+  raw_string_ostream Output(LambdaStr);
+  toMustacheString(Return, Output);
+  Parser P = Parser(LambdaStr);
+  AstPtr LambdaNode = P.parse(Partials, Lambdas, SectionLambdas, Escapes);
+  LambdaNode->render(Contexts, OS);
+  return;
+}
+
+void Template::render(const json::Value &Data, llvm::raw_ostream &OS) {
+  Tree->render(Data, OS);
+}
+
+void Template::registerPartial(std::string Name, std::string Partial) {
+  Parser P = Parser(Partial);
+  AstPtr PartialTree = P.parse(Partials, Lambdas, SectionLambdas, Escapes);
+  Partials.insert(std::make_pair(Name, std::move(PartialTree)));
+}
+
+void Template::registerLambda(std::string Name, Lambda L) { Lambdas[Name] = L; }
+
+void Template::registerLambda(std::string Name, SectionLambda L) {
+  SectionLambdas[Name] = L;
+}
+
+void Template::overrideEscapeCharacters(DenseMap<char, std::string> E) {
+  Escapes = std::move(E);
+}
+
+Template::Template(StringRef TemplateStr) {
+  Parser P = Parser(TemplateStr);
+  Tree = P.parse(Partials, Lambdas, SectionLambdas, Escapes);
+  // The default behavior is to escape html entities.
+  DenseMap<char, std::string> HtmlEntities = {{'&', "&amp;"},
+                                              {'<', "&lt;"},
+                                              {'>', "&gt;"},
+                                              {'"', "&quot;"},
+                                              {'\'', "&#39;"}};
+  overrideEscapeCharacters(HtmlEntities);
+}
+
+Template::Template(Template &&Other) noexcept
+    : Partials(std::move(Other.Partials)), Lambdas(std::move(Other.Lambdas)),
+      SectionLambdas(std::move(Other.SectionLambdas)),
+      Escapes(std::move(Other.Escapes)), Tree(std::move(Other.Tree)) {}
+
+Template::~Template() = default;
+
+Template &Template::operator=(Template &&Other) noexcept {
+  if (this != &Other) {
+    Partials = std::move(Other.Partials);
+    Lambdas = std::move(Other.Lambdas);
+    SectionLambdas = std::move(Other.SectionLambdas);
+    Escapes = std::move(Other.Escapes);
+    Tree = std::move(Other.Tree);
+    Other.Tree = nullptr;
+  }
+  return *this;
+}
+} // namespace llvm::mustache
diff --git a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp
index 5508af40663b1..c7a130a1f9c8a 100644
--- a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp
+++ b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp
@@ -186,51 +186,71 @@ void DXContainerGlobals::addResourcesForPSV(Module &M, PSVRuntimeInfo &PSV) {
   DXILResourceTypeMap &DRTM =
       getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
 
-  for (const dxil::ResourceBindingInfo &RBI : DBM) {
+  auto MakeBinding =
+      [](const dxil::ResourceBindingInfo::ResourceBinding &Binding,
+         const dxbc::PSV::ResourceType Type, const dxil::ResourceKind Kind,
+         const dxbc::PSV::ResourceFlags Flags = dxbc::PSV::ResourceFlags()) {
+        dxbc::PSV::v2::ResourceBindInfo BindInfo;
+        BindInfo.Type = Type;
+        BindInfo.LowerBound = Binding.LowerBound;
+        BindInfo.UpperBound = Binding.LowerBound + Binding.Size - 1;
+        BindInfo.Space = Binding.Space;
+        BindInfo.Kind = static_cast<dxbc::PSV::ResourceKind>(Kind);
+        BindInfo.Flags = Flags;
+        return BindInfo;
+      };
+
+  for (const dxil::ResourceBindingInfo &RBI : DBM.cbuffers()) {
+    const dxil::ResourceBindingInfo::ResourceBinding &Binding =
+        RBI.getBinding();
+    PSV.Resources.push_back(MakeBinding(Binding, dxbc::PSV::ResourceType::CBV,
+                                        dxil::ResourceKind::CBuffer));
+  }
+  for (const dxil::ResourceBindingInfo &RBI : DBM.samplers()) {
+    const dxil::ResourceBindingInfo::ResourceBinding &Binding =
+        RBI.getBinding();
+    PSV.Resources.push_back(MakeBinding(Binding,
+                                        dxbc::PSV::ResourceType::Sampler,
+                                        dxil::ResourceKind::Sampler));
+  }
+  for (const dxil::ResourceBindingInfo &RBI : DBM.srvs()) {
     const dxil::ResourceBindingInfo::ResourceBinding &Binding =
         RBI.getBinding();
-    dxbc::PSV::v2::ResourceBindInfo BindInfo;
-    BindInfo.LowerBound = Binding.LowerBound;
-    BindInfo.UpperBound = Binding.LowerBound + Binding.Size - 1;
-    BindInfo.Space = Binding.Space;
 
     dxil::ResourceTypeInfo &TypeInfo = DRTM[RBI.getHandleTy()];
-    dxbc::PSV::ResourceType ResType = dxbc::PSV::ResourceType::Invalid;
-    bool IsUAV = TypeInfo.getResourceClass() == dxil::ResourceClass::UAV;
-    switch (TypeInfo.getResourceKind()) {
-    case dxil::ResourceKind::Sampler:
-      ResType = dxbc::PSV::ResourceType::Sampler;
-      break;
-    case dxil::ResourceKind::CBuffer:
-      ResType = dxbc::PSV::ResourceType::CBV;
-      break;
-    case dxil::ResourceKind::StructuredBuffer:
-      ResType = IsUAV ? dxbc::PSV::ResourceType::UAVStructured
-                      : dxbc::PSV::ResourceType::SRVStructured;
-      if (IsUAV && TypeInfo.getUAV().HasCounter)
-        ResType = dxbc::PSV::ResourceType::UAVStructuredWithCounter;
-      break;
-    case dxil::ResourceKind::RTAccelerationStructure:
+    dxbc::PSV::ResourceType ResType;
+    if (TypeInfo.isStruct())
+      ResType = dxbc::PSV::ResourceType::SRVStructured;
+    else if (TypeInfo.isTyped())
+      ResType = dxbc::PSV::ResourceType::SRVTyped;
+    else
       ResType = dxbc::PSV::ResourceType::SRVRaw;
-      break;
-    case dxil::ResourceKind::RawBuffer:
-      ResType = IsUAV ? dxbc::PSV::ResourceType::UAVRaw
-                      : dxbc::PSV::ResourceType::SRVRaw;
-      break;
-    default:
-      ResType = IsUAV ? dxbc::PSV::ResourceType::UAVTyped
-                      : dxbc::PSV::ResourceType::SRVTyped;
-      break;
-    }
-    BindInfo.Type = ResType;
-
-    BindInfo.Kind =
-        static_cast<dxbc::PSV::ResourceKind>(TypeInfo.getResourceKind());
+
+    PSV.Resources.push_back(
+        MakeBinding(Binding, ResType, TypeInfo.getResourceKind()));
+  }
+  for (const dxil::ResourceBindingInfo &RBI : DBM.uavs()) {
+    const dxil::ResourceBindingInfo::ResourceBinding &Binding =
+        RBI.getBinding();
+
+    dxil::ResourceTypeInfo &TypeInfo = DRTM[RBI.getHandleTy()];
+    dxbc::PSV::ResourceType ResType;
+    if (TypeInfo.getUAV().HasCounter)
+      ResType = dxbc::PSV::ResourceType::UAVStructuredWithCounter;
+    else if (TypeInfo.isStruct())
+      ResType = dxbc::PSV::ResourceType::UAVStructured;
+    else if (TypeInfo.isTyped())
+      ResType = dxbc::PSV::ResourceType::UAVTyped;
+    else
+      ResType = dxbc::PSV::ResourceType::UAVRaw;
+
+    dxbc::PSV::ResourceFlags Flags;
     // TODO: Add support for dxbc::PSV::ResourceFlag::UsedByAtomic64, tracking
     // with https://github.com/llvm/llvm-project/issues/104392
-    BindInfo.Flags.Flags = 0u;
+    Flags.Flags = 0u;
 
-    PSV.Resources.emplace_back(BindInfo);
+    PSV.Resources.push_back(
+        MakeBinding(Binding, ResType, TypeInfo.getResourceKind(), Flags));
   }
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 95d697bbd734a..84ed1c11df9e0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -598,6 +598,28 @@ static std::optional<unsigned> getElementIndex(const Value *Inst,
   return Index;
 }
 
+/// \returns true if all of the values in \p VL use the same opcode.
+/// For comparison instructions, also checks if predicates match.
+/// PoisonValues are considered matching.
+/// Interchangeable instructions are not considered.
+static bool allSameOpcode(ArrayRef<Value *> VL) {
+  auto *It = find_if(VL, IsaPred<Instruction>);
+  if (It == VL.end())
+    return true;
+  Instruction *MainOp = cast<Instruction>(*It);
+  unsigned Opcode = MainOp->getOpcode();
+  bool IsCmpOp = isa<CmpInst>(MainOp);
+  CmpInst::Predicate BasePred = IsCmpOp ? cast<CmpInst>(MainOp)->getPredicate()
+                                        : CmpInst::BAD_ICMP_PREDICATE;
+  return std::all_of(It, VL.end(), [&](Value *V) {
+    if (auto *CI = dyn_cast<CmpInst>(V))
+      return BasePred == CI->getPredicate();
+    if (auto *I = dyn_cast<Instruction>(V))
+      return I->getOpcode() == Opcode;
+    return isa<PoisonValue>(V);
+  });
+}
+
 namespace {
 /// Specifies the way the mask should be analyzed for undefs/poisonous elements
 /// in the shuffle mask.
@@ -813,6 +835,301 @@ static std::optional<unsigned> getExtractIndex(const Instruction *E) {
 }
 
 namespace {
+/// \returns true if \p Opcode is allowed as part of the main/alternate
+/// instruction for SLP vectorization.
+///
+/// Example of unsupported opcode is SDIV that can potentially cause UB if the
+/// "shuffled out" lane would result in division by zero.
+bool isValidForAlternation(unsigned Opcode) {
+  return !Instruction::isIntDivRem(Opcode);
+}
+
+/// Helper class that determines VL can use the same opcode.
+/// Alternate instruction is supported. In addition, it supports interchangeable
+/// instruction. An interchangeable instruction is an instruction that can be
+/// converted to another instruction with same semantics. For example, x << 1 is
+/// equal to x * 2. x * 1 is equal to x | 0.
+class BinOpSameOpcodeHelper {
+  using MaskType = std::uint_fast16_t;
+  /// Sort SupportedOp because it is used by binary_search.
+  constexpr static std::initializer_list<unsigned> SupportedOp = {
+      Instruction::Add,  Instruction::Sub, Instruction::Mul, Instruction::Shl,
+      Instruction::AShr, Instruction::And, Instruction::Or,  Instruction::Xor};
+  enum : MaskType {
+    ShlBIT = 0b1,
+    AShrBIT = 0b10,
+    MulBIT = 0b100,
+    AddBIT = 0b1000,
+    SubBIT = 0b10000,
+    AndBIT = 0b100000,
+    OrBIT = 0b1000000,
+    XorBIT = 0b10000000,
+    MainOpBIT = 0b100000000,
+    LLVM_MARK_AS_BITMASK_ENUM(MainOpBIT)
+  };
+  /// Return a non-nullptr if either operand of I is a ConstantInt.
+  /// The second return value represents the operand position. We check the
+  /// right-hand side first (1). If the right hand side is not a ConstantInt and
+  /// the instruction is neither Sub, Shl, nor AShr, we then check the left hand
+  /// side (0).
+  static std::pair<ConstantInt *, unsigned>
+  isBinOpWithConstantInt(Instruction *I) {
+    unsigned Opcode = I->getOpcode();
+    assert(binary_search(SupportedOp, Opcode) && "Unsupported opcode.");
+    (void)SupportedOp;
+    auto *BinOp = cast<BinaryOperator>(I);
+    if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)))
+      return {CI, 1};
+    if (Opcode == Instruction::Sub || Opcode == Instruction::Shl ||
+        Opcode == Instruction::AShr)
+      return {nullptr, 0};
+    if (auto *CI = dyn_cast<ConstantInt>(BinOp->getOperand(0)))
+      return {CI, 0};
+    return {nullptr, 0};
+  }
+  struct InterchangeableInfo {
+    Instruction *I = nullptr;
+    /// The bit it sets represents whether MainOp can be converted to.
+    MaskType Mask = MainOpBIT | XorBIT | OrBIT | AndBIT | SubBIT | AddBIT |
+                    MulBIT | AShrBIT | ShlBIT;
+    /// We cannot create an interchangeable instruction that does not exist in
+    /// VL. For example, VL [x + 0, y * 1] can be converted to [x << 0, y << 0],
+    /// but << does not exist in VL. In the end, we convert VL to [x * 1, y *
+    /// 1]. SeenBefore is used to know what operations have been seen before.
+    MaskType SeenBefore = 0;
+    InterchangeableInfo(Instruction *I) : I(I) {}
+    /// Return false allows BinOpSameOpcodeHelper to find an alternate
+    /// instruction. Directly setting the mask will destroy the mask state,
+    /// preventing us from determining which instruction it should convert to.
+    bool trySet(MaskType OpcodeInMaskForm, MaskType InterchangeableMask) {
+      if (Mask & InterchangeableMask) {
+        SeenBefore |= OpcodeInMaskForm;
+        Mask &= InterchangeableMask;
+        return true;
+      }
+      return false;
+    }
+    bool equal(unsigned Opcode) {
+      if (Opcode == I->getOpcode())
+        return trySet(MainOpBIT, MainOpBIT);
+      return false;
+    }
+    unsigned getOpcode() const {
+      MaskType Candidate = Mask & SeenBefore;
+      if (Candidate & MainOpBIT)
+        return I->getOpcode();
+      if (Candidate & ShlBIT)
+        return Instruction::Shl;
+      if (Candidate & AShrBIT)
+        return Instruction::AShr;
+      if (Candidate & MulBIT)
+        return Instruction::Mul;
+      if (Candidate & AddBIT)
+        return Instruction::Add;
+      if (Candidate & SubBIT)
+        return Instruction::Sub;
+      if (Candidate & AndBIT)
+        return Instruction::And;
+      if (Candidate & OrBIT)
+        return Instruction::Or;
+      if (Candidate & XorBIT)
+        return Instruction::Xor;
+      llvm_unreachable("Cannot find interchangeable instruction.");
+    }
+    SmallVector<Value *> getOperand(Instruction *To) const {
+      unsigned ToOpcode = To->getOpcode();
+      unsigned FromOpcode = I->getOpcode();
+      if (FromOpcode == ToOpcode)
+        return SmallVector<Value *>(I->operands());
+      assert(binary_search(SupportedOp, ToOpcode) && "Unsupported opcode.");
+      auto [CI, Pos] = isBinOpWithConstantInt(I);
+      const APInt &FromCIValue = CI->getValue();
+      unsigned FromCIValueBitWidth = FromCIValue.getBitWidth();
+      APInt ToCIValue;
+      switch (FromOpcode) {
+      case Instruction::Shl:
+        if (ToOpcode == Instruction::Mul) {
+          ToCIValue = APInt::getOneBitSet(FromCIValueBitWidth,
+                                          FromCIValue.getZExtValue());
+        } else {
+          assert(FromCIValue.isZero() && "Cannot convert the instruction.");
+          ToCIValue = ToOpcode == Instruction::And
+                          ? APInt::getAllOnes(FromCIValueBitWidth)
+                          : APInt::getZero(FromCIValueBitWidth);
+        }
+        break;
+      case Instruction::Mul:
+        assert(FromCIValue.isPowerOf2() && "Cannot convert the instruction.");
+        if (ToOpcode == Instruction::Shl) {
+          ToCIValue = APInt(FromCIValueBitWidth, FromCIValue.logBase2());
+        } else {
+          assert(FromCIValue.isOne() && "Cannot convert the instruction.");
+          ToCIValue = ToOpcode == Instruction::And
+                          ? APInt::getAllOnes(FromCIValueBitWidth)
+                          : APInt::getZero(FromCIValueBitWidth);
+        }
+        break;
+      case Instruction::Add:
+      case Instruction::Sub:
+        if (FromCIValue.isZero()) {
+          ToCIValue = APInt::getZero(FromCIValueBitWidth);
+        } else {
+          assert(is_contained({Instruction::Add, Instruction::Sub}, ToOpcode) &&
+                 "Cannot convert the instruction.");
+          ToCIValue = FromCIValue;
+          ToCIValue.negate();
+        }
+        break;
+      case Instruction::And:
+        assert(FromCIValue.isAllOnes() && "Cannot convert the instruction.");
+        ToCIValue = ToOpcode == Instruction::Mul
+                        ? APInt::getOneBitSet(FromCIValueBitWidth, 0)
+                        : APInt::getZero(FromCIValueBitWidth);
+        break;
+      default:
+        assert(FromCIValue.isZero() && "Cannot convert the instruction.");
+        ToCIValue = APInt::getZero(FromCIValueBitWidth);
+        break;
+      }
+      Value *LHS = I->getOperand(1 - Pos);
+      Constant *RHS =
+          ConstantInt::get(I->getOperand(Pos)->getType(), ToCIValue);
+      if (Pos == 1)
+        return SmallVector<Value *>({LHS, RHS});
+      return SmallVector<Value *>({RHS, LHS});
+    }
+  };
+  InterchangeableInfo MainOp;
+  InterchangeableInfo AltOp;
+  bool isValidForAlternation(Instruction *I) const {
+    return ::isValidForAlternation(MainOp.I->getOpcode()) &&
+           ::isValidForAlternation(I->getOpcode());
+  }
+  bool initializeAltOp(Instruction *I) {
+    if (AltOp.I)
+      return true;
+    if (!isValidForAlternation(I))
+      return false;
+    AltOp.I = I;
+    return true;
+  }
+
+public:
+  BinOpSameOpcodeHelper(Instruction *MainOp, Instruction *AltOp = nullptr)
+      : MainOp(MainOp), AltOp(AltOp) {
+    assert(is_sorted(SupportedOp) && "SupportedOp is not sorted.");
+  }
+  bool add(Instruction *I) {
+    assert(isa<BinaryOperator>(I) &&
+           "BinOpSameOpcodeHelper only accepts BinaryOperator.");
+    unsigned Opcode = I->getOpcode();
+    MaskType OpcodeInMaskForm;
+    // Prefer Shl, AShr, Mul, Add, Sub, And, Or and Xor over MainOp.
+    switch (Opcode) {
+    case Instruction::Shl:
+      OpcodeInMaskForm = ShlBIT;
+      break;
+    case Instruction::AShr:
+      OpcodeInMaskForm = AShrBIT;
+      break;
+    case Instruction::Mul:
+      OpcodeInMaskForm = MulBIT;
+      break;
+    case Instruction::Add:
+      OpcodeInMaskForm = AddBIT;
+      break;
+    case Instruction::Sub:
+      OpcodeInMaskForm = SubBIT;
+      break;
+    case Instruction::And:
+      OpcodeInMaskForm = AndBIT;
+      break;
+    case Instruction::Or:
+      OpcodeInMaskForm = OrBIT;
+      break;
+    case Instruction::Xor:
+      OpcodeInMaskForm = XorBIT;
+      break;
+    default:
+      return MainOp.equal(Opcode) ||
+             (initializeAltOp(I) && AltOp.equal(Opcode));
+    }
+    MaskType InterchangeableMask = OpcodeInMaskForm;
+    ConstantInt *CI = isBinOpWithConstantInt(I).first;
+    if (CI) {
+      constexpr MaskType CanBeAll =
+          XorBIT | OrBIT | AndBIT | SubBIT | AddBIT | MulBIT | AShrBIT | ShlBIT;
+      const APInt &CIValue = CI->getValue();
+      switch (Opcode) {
+      case Instruction::Shl:
+        InterchangeableMask = CIValue.isZero() ? CanBeAll : MulBIT | ShlBIT;
+        break;
+      case Instruction::Mul:
+        if (CIValue.isOne()) {
+          InterchangeableMask = CanBeAll;
+          break;
+        }
+        if (CIValue.isPowerOf2())
+          InterchangeableMask = MulBIT | ShlBIT;
+        break;
+      case Instruction::Add:
+      case Instruction::Sub:
+        InterchangeableMask = CIValue.isZero() ? CanBeAll : SubBIT | AddBIT;
+        break;
+      case Instruction::And:
+        if (CIValue.isAllOnes())
+          InterchangeableMask = CanBeAll;
+        break;
+      default:
+        if (CIValue.isZero())
+          InterchangeableMask = CanBeAll;
+        break;
+      }
+    }
+    return MainOp.trySet(OpcodeInMaskForm, InterchangeableMask) ||
+           (initializeAltOp(I) &&
+            AltOp.trySet(OpcodeInMaskForm, InterchangeableMask));
+  }
+  unsigned getMainOpcode() const { return MainOp.getOpcode(); }
+  bool hasAltOp() const { return AltOp.I; }
+  unsigned getAltOpcode() const {
+    return hasAltOp() ? AltOp.getOpcode() : getMainOpcode();
+  }
+  SmallVector<Value *> getMainOperand(Instruction *I) const {
+    return MainOp.getOperand(I);
+  }
+  SmallVector<Value *> getAltOperand(Instruction *I) const {
+    return AltOp.getOperand(I);
+  }
+};
+
+bool isConvertible(Instruction *I, Instruction *MainOp, Instruction *AltOp) {
+  assert(MainOp && "MainOp cannot be nullptr.");
+  if (I->getOpcode() == MainOp->getOpcode())
+    return true;
+  assert(AltOp && "AltOp cannot be nullptr.");
+  if (I->getOpcode() == AltOp->getOpcode())
+    return true;
+  if (!I->isBinaryOp())
+    return false;
+  BinOpSameOpcodeHelper Converter(MainOp, AltOp);
+  return Converter.add(I) && Converter.add(MainOp) && Converter.add(AltOp);
+}
+
+std::pair<Instruction *, SmallVector<Value *>>
+convertTo(Instruction *I, Instruction *MainOp, Instruction *AltOp) {
+  assert(isConvertible(I, MainOp, AltOp) && "Cannot convert the instruction.");
+  if (I->getOpcode() == MainOp->getOpcode())
+    return std::make_pair(MainOp, SmallVector<Value *>(I->operands()));
+  // Prefer AltOp instead of interchangeable instruction of MainOp.
+  if (I->getOpcode() == AltOp->getOpcode())
+    return std::make_pair(AltOp, SmallVector<Value *>(I->operands()));
+  assert(I->isBinaryOp() && "Cannot convert the instruction.");
+  BinOpSameOpcodeHelper Converter(I);
+  if (Converter.add(I) && Converter.add(MainOp) && !Converter.hasAltOp())
+    return std::make_pair(MainOp, Converter.getMainOperand(MainOp));
+  return std::make_pair(AltOp, Converter.getAltOperand(AltOp));
+}
 
 /// Main data required for vectorization of instructions.
 class InstructionsState {
@@ -840,8 +1157,7 @@ class InstructionsState {
   bool isAltShuffle() const { return getMainOp() != getAltOp(); }
 
   bool isOpcodeOrAlt(Instruction *I) const {
-    unsigned CheckedOpcode = I->getOpcode();
-    return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
+    return isConvertible(I, MainOp, AltOp);
   }
 
   /// Checks if main/alt instructions are shift operations.
@@ -886,18 +1202,6 @@ class InstructionsState {
 
 } // end anonymous namespace
 
-/// \returns true if \p Opcode is allowed as part of the main/alternate
-/// instruction for SLP vectorization.
-///
-/// Example of unsupported opcode is SDIV that can potentially cause UB if the
-/// "shuffled out" lane would result in division by zero.
-static bool isValidForAlternation(unsigned Opcode) {
-  if (Instruction::isIntDivRem(Opcode))
-    return false;
-
-  return true;
-}
-
 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
                                        const TargetLibraryInfo &TLI);
 
@@ -955,6 +1259,17 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
       (VL.size() == 2 && InstCnt < 2))
     return InstructionsState::invalid();
 
+  auto FindInstructionWithOpcode = [&](unsigned Opcode) {
+    for (Value *V : VL) {
+      if (isa<PoisonValue>(V))
+        continue;
+      auto *Inst = cast<Instruction>(V);
+      if (Inst->getOpcode() == Opcode)
+        return Inst;
+    }
+    llvm_unreachable("Opcode not found.");
+  };
+
   bool IsCastOp = isa<CastInst>(MainOp);
   bool IsBinOp = isa<BinaryOperator>(MainOp);
   bool IsCmpOp = isa<CmpInst>(MainOp);
@@ -964,6 +1279,7 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
   unsigned Opcode = MainOp->getOpcode();
   unsigned AltOpcode = Opcode;
 
+  BinOpSameOpcodeHelper BinOpHelper(MainOp);
   bool SwappedPredsCompatible = IsCmpOp && [&]() {
     SetVector<unsigned> UniquePreds, UniqueNonSwappedPreds;
     UniquePreds.insert(BasePred);
@@ -1010,14 +1326,8 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
       return InstructionsState::invalid();
     unsigned InstOpcode = I->getOpcode();
     if (IsBinOp && isa<BinaryOperator>(I)) {
-      if (InstOpcode == Opcode || InstOpcode == AltOpcode)
+      if (BinOpHelper.add(I))
         continue;
-      if (Opcode == AltOpcode && isValidForAlternation(InstOpcode) &&
-          isValidForAlternation(Opcode)) {
-        AltOpcode = InstOpcode;
-        AltOp = I;
-        continue;
-      }
     } else if (IsCastOp && isa<CastInst>(I)) {
       Value *Op0 = MainOp->getOperand(0);
       Type *Ty0 = Op0->getType();
@@ -1118,6 +1428,12 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
     return InstructionsState::invalid();
   }
 
+  if (IsBinOp) {
+    MainOp = FindInstructionWithOpcode(BinOpHelper.getMainOpcode());
+    AltOp = FindInstructionWithOpcode(BinOpHelper.getAltOpcode());
+  }
+  assert((MainOp == AltOp || !allSameOpcode(VL)) &&
+         "Incorrect implementation of allSameOpcode.");
   return InstructionsState(MainOp, AltOp);
 }
 
@@ -2520,11 +2836,12 @@ class BoUpSLP {
         // Since operand reordering is performed on groups of commutative
         // operations or alternating sequences (e.g., +, -), we can safely tell
         // the inverse operations by checking commutativity.
-        bool IsInverseOperation = !isCommutative(cast<Instruction>(V));
+        auto [SelectedOp, Ops] =
+            convertTo(cast<Instruction>(VL[Lane]), MainOp, S.getAltOp());
+        bool IsInverseOperation = !isCommutative(SelectedOp);
         for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
           bool APO = (OpIdx == 0) ? false : IsInverseOperation;
-          OpsVec[OpIdx][Lane] = {cast<Instruction>(V)->getOperand(OpIdx), APO,
-                                 false};
+          OpsVec[OpIdx][Lane] = {Ops[OpIdx], APO, false};
         }
       }
     }
@@ -10735,7 +11052,9 @@ void BoUpSLP::transformNodes() {
       // same opcode and same parent block or all constants.
       if (VL.size() <= 2 || LoadEntriesToVectorize.contains(Idx) ||
           !(!E.hasState() || E.getOpcode() == Instruction::Load ||
-            E.isAltShuffle() || !allSameBlock(VL)) ||
+            // We use allSameOpcode instead of isAltShuffle because we don't
+            // want to use interchangeable instruction here.
+            !allSameOpcode(VL) || !allSameBlock(VL)) ||
           allConstant(VL) || isSplat(VL))
         continue;
       if (ForceLoadGather && E.hasState() && E.getOpcode() == Instruction::Load)
@@ -10780,7 +11099,7 @@ void BoUpSLP::transformNodes() {
             if (IsSplat)
               continue;
             InstructionsState S = getSameOpcode(Slice, *TLI);
-            if (!S || S.isAltShuffle() || !allSameBlock(Slice) ||
+            if (!S || !allSameOpcode(Slice) || !allSameBlock(Slice) ||
                 (S.getOpcode() == Instruction::Load &&
                  areKnownNonVectorizableLoads(Slice)) ||
                 (S.getOpcode() != Instruction::Load &&
@@ -12583,14 +12902,22 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       if (isa<PoisonValue>(UniqueValues[Idx]))
         return InstructionCost(TTI::TCC_Free);
 
-      auto *VI = cast<Instruction>(UniqueValues[Idx]);
-      unsigned OpIdx = isa<UnaryOperator>(VI) ? 0 : 1;
-      TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(VI->getOperand(0));
-      TTI::OperandValueInfo Op2Info =
-          TTI::getOperandInfo(VI->getOperand(OpIdx));
-      SmallVector<const Value *> Operands(VI->operand_values());
+      // We cannot retrieve the operand from UniqueValues[Idx] because an
+      // interchangeable instruction may be used. The order and the actual
+      // operand might differ from what is retrieved from UniqueValues[Idx].
+      Value *Op1 = E->getOperand(0)[Idx];
+      Value *Op2;
+      SmallVector<const Value *, 2> Operands(1, Op1);
+      if (isa<UnaryOperator>(UniqueValues[Idx])) {
+        Op2 = Op1;
+      } else {
+        Op2 = E->getOperand(1)[Idx];
+        Operands.push_back(Op2);
+      }
+      TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(Op1);
+      TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(Op2);
       return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind,
-                                         Op1Info, Op2Info, Operands, VI);
+                                         Op1Info, Op2Info, Operands);
     };
     auto GetVectorCost = [=](InstructionCost CommonCost) {
       if (ShuffleOrOp == Instruction::And && It != MinBWs.end()) {
@@ -17071,7 +17398,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       Value *V = Builder.CreateBinOp(
           static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS,
           RHS);
-      propagateIRFlags(V, E->Scalars, VL0, It == MinBWs.end());
+      propagateIRFlags(V, E->Scalars, nullptr, It == MinBWs.end());
       if (auto *I = dyn_cast<Instruction>(V)) {
         V = ::propagateMetadata(I, E->Scalars);
         // Drop nuw flags for abs(sub(commutative), true).
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 433fb247754bc..a7c85d30ba9f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -532,7 +532,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
     case VPRecipeBase::VPWidenPointerInductionSC:
     case VPRecipeBase::VPReductionPHISC:
     case VPRecipeBase::VPScalarCastSC:
-    case VPRecipeBase::VPScalarPHISC:
     case VPRecipeBase::VPPartialReductionSC:
       return true;
     case VPRecipeBase::VPBranchOnMaskSC:
@@ -2819,8 +2818,8 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
   VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC)
 
   void execute(VPTransformState &State) override {
-    llvm_unreachable(
-        "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+    llvm_unreachable("cannot execute this recipe, should be replaced by a "
+                     "scalar phi recipe");
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2905,8 +2904,8 @@ class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
   VP_CLASSOF_IMPL(VPDef::VPEVLBasedIVPHISC)
 
   void execute(VPTransformState &State) override {
-    llvm_unreachable(
-        "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+    llvm_unreachable("cannot execute this recipe, should be replaced by a "
+                     "scalar phi recipe");
   }
 
   /// Return the cost of this VPEVLBasedIVPHIRecipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index d0fa62978c1b4..2b762d0533d19 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -360,7 +360,6 @@ class VPDef {
     VPFirstOrderRecurrencePHISC,
     VPWidenIntOrFpInductionSC,
     VPWidenPointerInductionSC,
-    VPScalarPHISC,
     VPReductionPHISC,
     // END: SubclassID for recipes that inherit VPHeaderPHIRecipe
     // END: Phi-like recipes
diff --git a/llvm/test/CodeGen/AArch64/complex-int-to-fp.ll b/llvm/test/CodeGen/AArch64/complex-int-to-fp.ll
index 506e5e59a3529..ec504b4782547 100644
--- a/llvm/test/CodeGen/AArch64/complex-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/complex-int-to-fp.ll
@@ -1,9 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
 
-; CHECK: autogen_SD19655
-; CHECK: scvtf
-; CHECK: ret
 define void @autogen_SD19655(ptr %addr, ptr %addrfloat) {
+; CHECK-LABEL: autogen_SD19655:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    scvtf.2d v0, v0
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    str d0, [x1]
+; CHECK-NEXT:    ret
   %T = load <2 x i64>, ptr %addr
   %F = sitofp <2 x i64> %T to <2 x float>
   store <2 x float> %F, ptr %addrfloat
@@ -12,38 +17,44 @@ define void @autogen_SD19655(ptr %addr, ptr %addrfloat) {
 
 define <2 x double> @test_signed_v2i32_to_v2f64(<2 x i32> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v2i32_to_v2f64:
-; CHECK: sshll.2d [[VAL64:v[0-9]+]], v0, #0
-; CHECK-NEXT: scvtf.2d v0, [[VAL64]]
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshll.2d v0, v0, #0
+; CHECK-NEXT:    scvtf.2d v0, v0
+; CHECK-NEXT:    ret
   %conv = sitofp <2 x i32> %v to <2 x double>
   ret <2 x double> %conv
 }
 
 define <2 x double> @test_unsigned_v2i32_to_v2f64(<2 x i32> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v2i32_to_v2f64
-; CHECK: ushll.2d [[VAL64:v[0-9]+]], v0, #0
-; CHECK-NEXT: ucvtf.2d v0, [[VAL64]]
-; CHECK-NEXT: ret
+; CHECK-LABEL: test_unsigned_v2i32_to_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll.2d v0, v0, #0
+; CHECK-NEXT:    ucvtf.2d v0, v0
+; CHECK-NEXT:    ret
   %conv = uitofp <2 x i32> %v to <2 x double>
   ret <2 x double> %conv
 }
 
 define <2 x double> @test_signed_v2i16_to_v2f64(<2 x i16> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v2i16_to_v2f64:
-; CHECK: shl.2s [[TMP:v[0-9]+]], v0, #16
-; CHECK: sshr.2s [[VAL32:v[0-9]+]], [[TMP]], #16
-; CHECK: sshll.2d [[VAL64:v[0-9]+]], [[VAL32]], #0
-; CHECK: scvtf.2d v0, [[VAL64]]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl.2s v0, v0, #16
+; CHECK-NEXT:    sshr.2s v0, v0, #16
+; CHECK-NEXT:    sshll.2d v0, v0, #0
+; CHECK-NEXT:    scvtf.2d v0, v0
+; CHECK-NEXT:    ret
 
   %conv = sitofp <2 x i16> %v to <2 x double>
   ret <2 x double> %conv
 }
 define <2 x double> @test_unsigned_v2i16_to_v2f64(<2 x i16> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v2i16_to_v2f64
-; CHECK: movi d[[MASK:[0-9]+]], #0x00ffff0000ffff
-; CHECK: and.8b [[VAL32:v[0-9]+]], v0, v[[MASK]]
-; CHECK: ushll.2d [[VAL64:v[0-9]+]], [[VAL32]], #0
-; CHECK: ucvtf.2d v0, [[VAL64]]
+; CHECK-LABEL: test_unsigned_v2i16_to_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-NEXT:    and.8b v0, v0, v1
+; CHECK-NEXT:    ushll.2d v0, v0, #0
+; CHECK-NEXT:    ucvtf.2d v0, v0
+; CHECK-NEXT:    ret
 
   %conv = uitofp <2 x i16> %v to <2 x double>
   ret <2 x double> %conv
@@ -51,20 +62,24 @@ define <2 x double> @test_unsigned_v2i16_to_v2f64(<2 x i16> %v) nounwind readnon
 
 define <2 x double> @test_signed_v2i8_to_v2f64(<2 x i8> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v2i8_to_v2f64:
-; CHECK: shl.2s [[TMP:v[0-9]+]], v0, #24
-; CHECK: sshr.2s [[VAL32:v[0-9]+]], [[TMP]], #24
-; CHECK: sshll.2d [[VAL64:v[0-9]+]], [[VAL32]], #0
-; CHECK: scvtf.2d v0, [[VAL64]]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl.2s v0, v0, #24
+; CHECK-NEXT:    sshr.2s v0, v0, #24
+; CHECK-NEXT:    sshll.2d v0, v0, #0
+; CHECK-NEXT:    scvtf.2d v0, v0
+; CHECK-NEXT:    ret
 
   %conv = sitofp <2 x i8> %v to <2 x double>
   ret <2 x double> %conv
 }
 define <2 x double> @test_unsigned_v2i8_to_v2f64(<2 x i8> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v2i8_to_v2f64
-; CHECK: movi d[[MASK:[0-9]+]], #0x0000ff000000ff
-; CHECK: and.8b [[VAL32:v[0-9]+]], v0, v[[MASK]]
-; CHECK: ushll.2d [[VAL64:v[0-9]+]], [[VAL32]], #0
-; CHECK: ucvtf.2d v0, [[VAL64]]
+; CHECK-LABEL: test_unsigned_v2i8_to_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NEXT:    and.8b v0, v0, v1
+; CHECK-NEXT:    ushll.2d v0, v0, #0
+; CHECK-NEXT:    ucvtf.2d v0, v0
+; CHECK-NEXT:    ret
 
   %conv = uitofp <2 x i8> %v to <2 x double>
   ret <2 x double> %conv
@@ -72,16 +87,20 @@ define <2 x double> @test_unsigned_v2i8_to_v2f64(<2 x i8> %v) nounwind readnone
 
 define <2 x float> @test_signed_v2i64_to_v2f32(<2 x i64> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v2i64_to_v2f32:
-; CHECK: scvtf.2d [[VAL64:v[0-9]+]], v0
-; CHECK: fcvtn v0.2s, [[VAL64]].2d
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    scvtf.2d v0, v0
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
 
   %conv = sitofp <2 x i64> %v to <2 x float>
   ret <2 x float> %conv
 }
 define <2 x float> @test_unsigned_v2i64_to_v2f32(<2 x i64> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v2i64_to_v2f32
-; CHECK: ucvtf.2d [[VAL64:v[0-9]+]], v0
-; CHECK: fcvtn v0.2s, [[VAL64]].2d
+; CHECK-LABEL: test_unsigned_v2i64_to_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ucvtf.2d v0, v0
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
 
   %conv = uitofp <2 x i64> %v to <2 x float>
   ret <2 x float> %conv
@@ -89,18 +108,22 @@ define <2 x float> @test_unsigned_v2i64_to_v2f32(<2 x i64> %v) nounwind readnone
 
 define <2 x float> @test_signed_v2i16_to_v2f32(<2 x i16> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v2i16_to_v2f32:
-; CHECK: shl.2s [[TMP:v[0-9]+]], v0, #16
-; CHECK: sshr.2s [[VAL32:v[0-9]+]], [[TMP]], #16
-; CHECK: scvtf.2s v0, [[VAL32]]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl.2s v0, v0, #16
+; CHECK-NEXT:    sshr.2s v0, v0, #16
+; CHECK-NEXT:    scvtf.2s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = sitofp <2 x i16> %v to <2 x float>
   ret <2 x float> %conv
 }
 define <2 x float> @test_unsigned_v2i16_to_v2f32(<2 x i16> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v2i16_to_v2f32
-; CHECK: movi d[[MASK:[0-9]+]], #0x00ffff0000ffff
-; CHECK: and.8b [[VAL32:v[0-9]+]], v0, v[[MASK]]
-; CHECK: ucvtf.2s v0, [[VAL32]]
+; CHECK-LABEL: test_unsigned_v2i16_to_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-NEXT:    and.8b v0, v0, v1
+; CHECK-NEXT:    ucvtf.2s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = uitofp <2 x i16> %v to <2 x float>
   ret <2 x float> %conv
@@ -108,18 +131,22 @@ define <2 x float> @test_unsigned_v2i16_to_v2f32(<2 x i16> %v) nounwind readnone
 
 define <2 x float> @test_signed_v2i8_to_v2f32(<2 x i8> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v2i8_to_v2f32:
-; CHECK: shl.2s [[TMP:v[0-9]+]], v0, #24
-; CHECK: sshr.2s [[VAL32:v[0-9]+]], [[TMP]], #24
-; CHECK: scvtf.2s v0, [[VAL32]]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl.2s v0, v0, #24
+; CHECK-NEXT:    sshr.2s v0, v0, #24
+; CHECK-NEXT:    scvtf.2s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = sitofp <2 x i8> %v to <2 x float>
   ret <2 x float> %conv
 }
 define <2 x float> @test_unsigned_v2i8_to_v2f32(<2 x i8> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v2i8_to_v2f32
-; CHECK: movi d[[MASK:[0-9]+]], #0x0000ff000000ff
-; CHECK: and.8b [[VAL32:v[0-9]+]], v0, v[[MASK]]
-; CHECK: ucvtf.2s v0, [[VAL32]]
+; CHECK-LABEL: test_unsigned_v2i8_to_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NEXT:    and.8b v0, v0, v1
+; CHECK-NEXT:    ucvtf.2s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = uitofp <2 x i8> %v to <2 x float>
   ret <2 x float> %conv
@@ -127,17 +154,21 @@ define <2 x float> @test_unsigned_v2i8_to_v2f32(<2 x i8> %v) nounwind readnone {
 
 define <4 x float> @test_signed_v4i16_to_v4f32(<4 x i16> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v4i16_to_v4f32:
-; CHECK: sshll.4s [[VAL32:v[0-9]+]], v0, #0
-; CHECK: scvtf.4s v0, [[VAL32]]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshll.4s v0, v0, #0
+; CHECK-NEXT:    scvtf.4s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = sitofp <4 x i16> %v to <4 x float>
   ret <4 x float> %conv
 }
 
 define <4 x float> @test_unsigned_v4i16_to_v4f32(<4 x i16> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v4i16_to_v4f32
-; CHECK: ushll.4s [[VAL32:v[0-9]+]], v0, #0
-; CHECK: ucvtf.4s v0, [[VAL32]]
+; CHECK-LABEL: test_unsigned_v4i16_to_v4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll.4s v0, v0, #0
+; CHECK-NEXT:    ucvtf.4s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = uitofp <4 x i16> %v to <4 x float>
   ret <4 x float> %conv
@@ -145,19 +176,23 @@ define <4 x float> @test_unsigned_v4i16_to_v4f32(<4 x i16> %v) nounwind readnone
 
 define <4 x float> @test_signed_v4i8_to_v4f32(<4 x i8> %v) nounwind readnone {
 ; CHECK-LABEL: test_signed_v4i8_to_v4f32:
-; CHECK: shl.4h [[TMP:v[0-9]+]], v0, #8
-; CHECK: sshr.4h [[VAL16:v[0-9]+]], [[TMP]], #8
-; CHECK: sshll.4s [[VAL32:v[0-9]+]], [[VAL16]], #0
-; CHECK: scvtf.4s v0, [[VAL32]]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shl.4h v0, v0, #8
+; CHECK-NEXT:    sshr.4h v0, v0, #8
+; CHECK-NEXT:    sshll.4s v0, v0, #0
+; CHECK-NEXT:    scvtf.4s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = sitofp <4 x i8> %v to <4 x float>
   ret <4 x float> %conv
 }
 define <4 x float> @test_unsigned_v4i8_to_v4f32(<4 x i8> %v) nounwind readnone {
-; CHECK-LABEL: test_unsigned_v4i8_to_v4f32
-; CHECK: bic.4h v0, #255, lsl #8
-; CHECK: ushll.4s [[VAL32:v[0-9]+]], v0, #0
-; CHECK: ucvtf.4s v0, [[VAL32]]
+; CHECK-LABEL: test_unsigned_v4i8_to_v4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic.4h v0, #255, lsl #8
+; CHECK-NEXT:    ushll.4s v0, v0, #0
+; CHECK-NEXT:    ucvtf.4s v0, v0
+; CHECK-NEXT:    ret
 
   %conv = uitofp <4 x i8> %v to <4 x float>
   ret <4 x float> %conv
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/PSVResources-order.ll b/llvm/test/CodeGen/DirectX/ContainerData/PSVResources-order.ll
new file mode 100644
index 0000000000000..734149eec598e
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/PSVResources-order.ll
@@ -0,0 +1,26 @@
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s
+
+; Check that resources are emitted to the object in the order that matches what
+; the DXIL validator expects: CBuffers, Samplers, SRVs, and then UAVs.
+
+; CHECK: Resources:
+; CHECK: - Type: CBV
+; TODO:  - Type: Sampler
+; CHECK: - Type: SRVRaw
+; CHECK: - Type: UAVTyped
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+define void @main() #0 {
+  %uav0 = call target("dx.TypedBuffer", i32, 1, 0, 1)
+      @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i32_1_0t(
+          i32 2, i32 7, i32 1, i32 0, i1 false)
+  %srv0 = call target("dx.RawBuffer", i8, 0, 0)
+      @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i8_0_0t(
+          i32 1, i32 8, i32 1, i32 0, i1 false)
+  %cbuf = call target("dx.CBuffer", target("dx.Layout", {float}, 4, 0))
+      @llvm.dx.resource.handlefrombinding(i32 3, i32 2, i32 1, i32 0, i1 false)
+  ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/PSVResources.ll b/llvm/test/CodeGen/DirectX/ContainerData/PSVResources.ll
index ce67812c3988f..cea8ba2f067c1 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/PSVResources.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/PSVResources.ll
@@ -6,6 +6,17 @@ target triple = "dxil-unknown-shadermodel6.0-compute"
 
 define void @main() #0 {
 
+  ; cbuffer : register(b2, space3) { float x; }
+; CHECK:        - Type:            CBV
+; CHECK:          Space:           3
+; CHECK:          LowerBound:      2
+; CHECK:          UpperBound:      2
+; CHECK:          Kind:            CBuffer
+; CHECK:          Flags:
+; CHECK:            UsedByAtomic64:  false
+  %cbuf = call target("dx.CBuffer", target("dx.Layout", {float}, 4, 0))
+      @llvm.dx.resource.handlefrombinding(i32 3, i32 2, i32 1, i32 0, i1 false)
+
   ; ByteAddressBuffer Buf : register(t8, space1)
 ; CHECK:        - Type:            SRVRaw
 ; CHECK:          Space:           1
diff --git a/llvm/test/CodeGen/DirectX/HLSLControlFlowHint.ll b/llvm/test/CodeGen/DirectX/HLSLControlFlowHint.ll
index 6a5274429930e..00dd374daf460 100644
--- a/llvm/test/CodeGen/DirectX/HLSLControlFlowHint.ll
+++ b/llvm/test/CodeGen/DirectX/HLSLControlFlowHint.ll
@@ -91,6 +91,137 @@ if.end:                                           ; preds = %if.else, %if.then
   %3 = load i32, ptr %resp, align 4
   ret i32 %3
 }
+
+; CHECK: define i32 @flatten_switch(i32 %X)
+; CHECK-NOT: hlsl.controlflow.hint
+; CHECK:      switch i32 %0, label %sw.epilog [
+; CHECK-NEXT:   i32 0, label %sw.bb
+; CHECK-NEXT:   i32 1, label %sw.bb1
+; CHECK-NEXT:   i32 2, label %sw.bb2
+; CHECK-NEXT: ], !dx.controlflow.hints [[HINT_FLATTEN]]
+define i32 @flatten_switch(i32 %X) #0 {
+entry:
+  %X.addr = alloca i32, align 4
+  %resp = alloca i32, align 4
+  store i32 %X, ptr %X.addr, align 4
+  %0 = load i32, ptr %X.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ], !hlsl.controlflow.hint !1
+
+sw.bb:                                            ; preds = %entry
+  %1 = load i32, ptr %X.addr, align 4
+  %sub = sub nsw i32 0, %1
+  store i32 %sub, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry
+  %2 = load i32, ptr %X.addr, align 4
+  %3 = load i32, ptr %X.addr, align 4
+  %add = add nsw i32 %2, %3
+  store i32 %add, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  %4 = load i32, ptr %X.addr, align 4
+  %5 = load i32, ptr %X.addr, align 4
+  %mul = mul nsw i32 %4, %5
+  store i32 %mul, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb2, %sw.bb1, %sw.bb
+  %6 = load i32, ptr %resp, align 4
+  ret i32 %6
+}
+
+
+; CHECK: define i32 @branch_switch(i32 %X)
+; CHECK-NOT: hlsl.controlflow.hint
+; CHECK:      switch i32 %0, label %sw.epilog [
+; CHECK-NEXT:   i32 0, label %sw.bb
+; CHECK-NEXT:   i32 1, label %sw.bb1
+; CHECK-NEXT:   i32 2, label %sw.bb2
+; CHECK-NEXT: ], !dx.controlflow.hints [[HINT_BRANCH]]
+define i32 @branch_switch(i32 %X) #0 {
+entry:
+  %X.addr = alloca i32, align 4
+  %resp = alloca i32, align 4
+  store i32 %X, ptr %X.addr, align 4
+  %0 = load i32, ptr %X.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ], !hlsl.controlflow.hint !0
+
+sw.bb:                                            ; preds = %entry
+  %1 = load i32, ptr %X.addr, align 4
+  %sub = sub nsw i32 0, %1
+  store i32 %sub, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry
+  %2 = load i32, ptr %X.addr, align 4
+  %3 = load i32, ptr %X.addr, align 4
+  %add = add nsw i32 %2, %3
+  store i32 %add, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  %4 = load i32, ptr %X.addr, align 4
+  %5 = load i32, ptr %X.addr, align 4
+  %mul = mul nsw i32 %4, %5
+  store i32 %mul, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb2, %sw.bb1, %sw.bb
+  %6 = load i32, ptr %resp, align 4
+  ret i32 %6
+}
+
+
+; CHECK: define i32 @no_attr_switch(i32 %X)
+; CHECK-NOT: hlsl.controlflow.hint
+; CHECK-NOT: !dx.controlflow.hints
+define i32 @no_attr_switch(i32 %X) #0 {
+entry:
+  %X.addr = alloca i32, align 4
+  %resp = alloca i32, align 4
+  store i32 %X, ptr %X.addr, align 4
+  %0 = load i32, ptr %X.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+
+sw.bb:                                            ; preds = %entry
+  %1 = load i32, ptr %X.addr, align 4
+  %sub = sub nsw i32 0, %1
+  store i32 %sub, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry
+  %2 = load i32, ptr %X.addr, align 4
+  %3 = load i32, ptr %X.addr, align 4
+  %add = add nsw i32 %2, %3
+  store i32 %add, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  %4 = load i32, ptr %X.addr, align 4
+  %5 = load i32, ptr %X.addr, align 4
+  %mul = mul nsw i32 %4, %5
+  store i32 %mul, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb2, %sw.bb1, %sw.bb
+  %6 = load i32, ptr %resp, align 4
+  ret i32 %6
+}
+
 ; CHECK-NOT: hlsl.controlflow.hint
 ; CHECK: [[HINT_BRANCH]] = !{!"dx.controlflow.hints", i32 1}
 ; CHECK: [[HINT_FLATTEN]] = !{!"dx.controlflow.hints", i32 2}
diff --git a/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll b/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll
index 848eaf70f5a19..9c6f977dc9b34 100644
--- a/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll
+++ b/llvm/test/CodeGen/SPIRV/structurizer/HLSLControlFlowHint.ll
@@ -5,7 +5,7 @@
 define spir_func noundef i32 @test_branch(i32 noundef %X) {
 entry:
 ; CHECK-LABEL: ; -- Begin function test_branch
-; OpSelectionMerge %[[#]] DontFlatten
+; CHECK: OpSelectionMerge %[[#]] DontFlatten
   %X.addr = alloca i32, align 4
   %resp = alloca i32, align 4
   store i32 %X, ptr %X.addr, align 4
@@ -34,7 +34,7 @@ if.end:                                           ; preds = %if.else, %if.then
 define spir_func noundef i32 @test_flatten(i32 noundef %X) {
 entry:
 ; CHECK-LABEL: ; -- Begin function test_flatten
-; OpSelectionMerge %[[#]] Flatten
+; CHECK: OpSelectionMerge %[[#]] Flatten
   %X.addr = alloca i32, align 4
   %resp = alloca i32, align 4
   store i32 %X, ptr %X.addr, align 4
@@ -62,7 +62,7 @@ if.end:                                           ; preds = %if.else, %if.then
 define spir_func noundef i32 @test_no_attr(i32 noundef %X) {
 entry:
 ; CHECK-LABEL: ; -- Begin function test_no_attr
-; OpSelectionMerge %[[#]] None
+; CHECK: OpSelectionMerge %[[#]] None
   %X.addr = alloca i32, align 4
   %resp = alloca i32, align 4
   store i32 %X, ptr %X.addr, align 4
@@ -87,5 +87,124 @@ if.end:                                           ; preds = %if.else, %if.then
   ret i32 %3
 }
 
+define spir_func noundef i32 @flatten_switch(i32 noundef %X) {
+entry:
+; CHECK-LABEL: ; -- Begin function flatten_switch
+; CHECK: OpSelectionMerge %[[#]] Flatten
+  %X.addr = alloca i32, align 4
+  %resp = alloca i32, align 4
+  store i32 %X, ptr %X.addr, align 4
+  %0 = load i32, ptr %X.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ], !hlsl.controlflow.hint !1
+
+sw.bb:                                            ; preds = %entry
+  %1 = load i32, ptr %X.addr, align 4
+  %sub = sub nsw i32 0, %1
+  store i32 %sub, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry
+  %2 = load i32, ptr %X.addr, align 4
+  %3 = load i32, ptr %X.addr, align 4
+  %add = add nsw i32 %2, %3
+  store i32 %add, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  %4 = load i32, ptr %X.addr, align 4
+  %5 = load i32, ptr %X.addr, align 4
+  %mul = mul nsw i32 %4, %5
+  store i32 %mul, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb2, %sw.bb1, %sw.bb
+  %6 = load i32, ptr %resp, align 4
+  ret i32 %6
+}
+
+
+define spir_func noundef i32 @branch_switch(i32 noundef %X) {
+  entry:
+  ; CHECK-LABEL: ; -- Begin function branch_switch
+  ; CHECK: OpSelectionMerge %[[#]] DontFlatten
+  %X.addr = alloca i32, align 4
+  %resp = alloca i32, align 4
+  store i32 %X, ptr %X.addr, align 4
+  %0 = load i32, ptr %X.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ], !hlsl.controlflow.hint !0
+
+sw.bb:                                            ; preds = %entry
+  %1 = load i32, ptr %X.addr, align 4
+  %sub = sub nsw i32 0, %1
+  store i32 %sub, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry
+  %2 = load i32, ptr %X.addr, align 4
+  %3 = load i32, ptr %X.addr, align 4
+  %add = add nsw i32 %2, %3
+  store i32 %add, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  %4 = load i32, ptr %X.addr, align 4
+  %5 = load i32, ptr %X.addr, align 4
+  %mul = mul nsw i32 %4, %5
+  store i32 %mul, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb2, %sw.bb1, %sw.bb
+  %6 = load i32, ptr %resp, align 4
+  ret i32 %6
+}
+
+
+define spir_func noundef i32 @no_attr_switch(i32 noundef %X) {
+  ; CHECK-LABEL: ; -- Begin function no_attr_switch
+; CHECK: OpSelectionMerge %[[#]] None
+entry:
+  %X.addr = alloca i32, align 4
+  %resp = alloca i32, align 4
+  store i32 %X, ptr %X.addr, align 4
+  %0 = load i32, ptr %X.addr, align 4
+  switch i32 %0, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+
+sw.bb:                                            ; preds = %entry
+  %1 = load i32, ptr %X.addr, align 4
+  %sub = sub nsw i32 0, %1
+  store i32 %sub, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry
+  %2 = load i32, ptr %X.addr, align 4
+  %3 = load i32, ptr %X.addr, align 4
+  %add = add nsw i32 %2, %3
+  store i32 %add, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  %4 = load i32, ptr %X.addr, align 4
+  %5 = load i32, ptr %X.addr, align 4
+  %mul = mul nsw i32 %4, %5
+  store i32 %mul, ptr %resp, align 4
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb2, %sw.bb1, %sw.bb
+  %6 = load i32, ptr %resp, align 4
+  ret i32 %6
+}
+
 !0 = !{!"hlsl.controlflow.hint", i32 1}
 !1 = !{!"hlsl.controlflow.hint", i32 2}
diff --git a/llvm/test/TableGen/AcquireAtCycle.td b/llvm/test/TableGen/AcquireAtCycle.td
index d9d4ccef54769..1cd6489c2902d 100644
--- a/llvm/test/TableGen/AcquireAtCycle.td
+++ b/llvm/test/TableGen/AcquireAtCycle.td
@@ -26,6 +26,7 @@ def ResX2 : ProcResource<1>; // X2
 let OutOperandList = (outs), InOperandList = (ins) in {
   def Inst_A : Instruction;
   def Inst_B : Instruction;
+  def Inst_C : Instruction;
 }
 
 let CompleteModel = 0 in {
@@ -34,6 +35,7 @@ let CompleteModel = 0 in {
 
 def WriteInst_A : SchedWrite;
 def WriteInst_B : SchedWrite;
+def WriteInst_C : SchedWrite;
 
 let SchedModel = SchedModel_A in {
 // Check the generated data when there are no semantic issues.
@@ -49,9 +51,14 @@ def : WriteRes<WriteInst_A, [ResX0, ResX1, ResX2]> {
 }
 def : WriteRes<WriteInst_B, [ResX2]> {
 // If unspecified, AcquireAtCycle is set to 0.
-// CORRECT-NEXT: { 3, 1, 0} // #4
+// CORRECT-NEXT: { 3, 1, 0}, // #4
     let ReleaseAtCycles = [1];
 }
+def : WriteRes<WriteInst_C, [ResX0]> {
+// AcquireAtCycle and ReleaseAtCycle are allowed to be the same.
+// CORRECT-NEXT: { 1, 0, 0} // #5
+    let ReleaseAtCycles = [0];
+}
 #endif // CORRECT
 
 #ifdef WRONG_SIZE
@@ -63,7 +70,7 @@ def : WriteRes<WriteInst_A, [ResX0, ResX1, ResX2]> {
 #endif
 
 #ifdef WRONG_VALUE
-// WRONG_VALUE: AcquireAtCycle.td:[[@LINE+1]]:1: error: Inconsistent resource cycles: AcquireAtCycles < ReleaseAtCycles must hold
+// WRONG_VALUE: AcquireAtCycle.td:[[@LINE+1]]:1: error: Inconsistent resource cycles: AcquireAtCycles <= ReleaseAtCycles must hold
 def : WriteRes<WriteInst_A, [ResX0, ResX1, ResX2]> {
     let ReleaseAtCycles = [2, 4, 3];
     let AcquireAtCycles = [0, 1, 8];
@@ -80,6 +87,7 @@ def : WriteRes<WriteInst_A, [ResX0]> {
 
 def : InstRW<[WriteInst_A], (instrs Inst_A)>;
 def : InstRW<[WriteInst_B], (instrs Inst_B)>;
+def : InstRW<[WriteInst_C], (instrs Inst_C)>;
 }
 
 def ProcessorA: ProcessorModel<"ProcessorA", SchedModel_A, []>;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
index feb4ad865f314..d527d38adbee3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
@@ -314,10 +314,10 @@ define void @store_try_reorder(ptr %dst) {
 ;
 ; POW2-ONLY-LABEL: @store_try_reorder(
 ; POW2-ONLY-NEXT:  entry:
-; POW2-ONLY-NEXT:    [[ADD:%.*]] = add i32 0, 0
-; POW2-ONLY-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
-; POW2-ONLY-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
-; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
+; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887:%.*]], align 4
+; POW2-ONLY-NEXT:    [[ADD216:%.*]] = sub i32 0, 0
+; POW2-ONLY-NEXT:    [[ARRAYIDX_I1891:%.*]] = getelementptr i32, ptr [[ARRAYIDX_I1887]], i64 2
+; POW2-ONLY-NEXT:    store i32 [[ADD216]], ptr [[ARRAYIDX_I1891]], align 4
 ; POW2-ONLY-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll
index fd3d4ab80b29c..ff897180cc9b7 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reversed-strided-node-with-external-ptr.ll
@@ -7,13 +7,12 @@ define void @test(ptr %a, i64 %0) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[A]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[TMP0]], i32 0
 ; CHECK-NEXT:    br label %[[BB:.*]]
 ; CHECK:       [[BB]]:
-; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[TMP4]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = or disjoint <2 x i64> [[TMP3]], <i64 1, i64 0>
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr double, <2 x ptr> [[TMP2]], <2 x i64> [[TMP5]]
-; CHECK-NEXT:    [[ARRAYIDX17_I28_1:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[ARRAYIDX17_I28_1:%.*]] = extractelement <2 x ptr> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> [[TMP6]], i32 8, <2 x i1> splat (i1 true), <2 x double> poison)
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x double>, ptr [[A]], align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x double>, ptr [[A]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
index 7ab5e4d6cb787..481d586e6658a 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
@@ -324,10 +324,10 @@ define void @store_try_reorder(ptr %dst) {
 ;
 ; POW2-ONLY-LABEL: @store_try_reorder(
 ; POW2-ONLY-NEXT:  entry:
-; POW2-ONLY-NEXT:    [[ADD:%.*]] = add i32 0, 0
-; POW2-ONLY-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
-; POW2-ONLY-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
-; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
+; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887:%.*]], align 4
+; POW2-ONLY-NEXT:    [[ADD216:%.*]] = sub i32 0, 0
+; POW2-ONLY-NEXT:    [[ARRAYIDX_I1891:%.*]] = getelementptr i32, ptr [[ARRAYIDX_I1887]], i64 2
+; POW2-ONLY-NEXT:    store i32 [[ADD216]], ptr [[ARRAYIDX_I1891]], align 4
 ; POW2-ONLY-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
index f46a5d84a86cc..a39e602e2da71 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
@@ -10,9 +10,7 @@ define i32 @foo(ptr nocapture %A, i32 %n) {
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @bar()
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i32 0
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 8, i32 10>
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], splat (i32 9)
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    ret i32 undef
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
index 889f5a95c81d6..299677ca80b34 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
@@ -10,15 +10,10 @@ define void @test(ptr %0, ptr %1, ptr %2) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = sub <4 x i32> <i32 0, i32 0, i32 undef, i32 0>, [[TMP8]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub <4 x i32> [[TMP11]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP6]]
-; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i32> [[TMP13]], <i32 0, i32 0, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP15:%.*]] = sub <4 x i32> [[TMP13]], <i32 0, i32 0, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> [[TMP15]], <4 x i32> <i32 2, i32 0, i32 1, i32 7>
+; CHECK-NEXT:    [[TMP16:%.*]] = add <4 x i32> <i32 0, i32 0, i32 1, i32 0>, [[TMP13]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = add <4 x i32> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP18:%.*]] = sub <4 x i32> [[TMP16]], zeroinitializer
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP17]], <4 x i32> [[TMP18]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP20:%.*]] = add <4 x i32> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP21:%.*]] = sub <4 x i32> [[TMP19]], zeroinitializer
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP20]], <4 x i32> [[TMP21]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = add <4 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    store <4 x i32> [[TMP22]], ptr [[TMP2:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
index 43c42c1ea2bfb..03a89e54e4212 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
@@ -8,15 +8,13 @@ define void @test() {
 ; CHECK:       [[BB1:.*]]:
 ; CHECK-NEXT:    br label %[[BB2:.*]]
 ; CHECK:       [[BB2]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x i32> [ poison, %[[BB1]] ], [ [[TMP5:%.*]], %[[BB6]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x i32> [ poison, %[[BB1]] ], [ [[TMP4:%.*]], %[[BB6]] ]
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[BB6]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP8:%.*]], %[[BB6]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 5, i32 4>
-; CHECK-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP5]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 5, i32 4>
+; CHECK-NEXT:    [[TMP4]] = mul <4 x i32> [[TMP3]], zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[TMP8]] = mul <2 x i32> zeroinitializer, [[TMP7]]
 ; CHECK-NEXT:    br i1 false, label %[[BB2]], label %[[BB6]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
index 766916fe71f35..c4ddc5d63cc04 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bv-shuffle-mask.ll
@@ -7,14 +7,12 @@ define i16 @test(i16 %v1, i16 %v2) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i16> <i16 0, i16 0, i16 0, i16 poison>, i16 [[V2]], i32 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> <i16 0, i16 0, i16 0, i16 poison>, i16 [[V1]], i32 3
-; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i16> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and <4 x i16> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 poison, i32 3>
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i16> [[TMP5]], i16 [[V1]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i16> [[TMP6]], <2 x i16> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i16> [[TMP7]], zeroinitializer
-; CHECK-NEXT:    [[TMP9:%.*]] = and <4 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = and <4 x i16> [[TMP3]], zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = and <4 x i16> [[TMP9]], zeroinitializer
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <4 x i16> [[TMP10]], zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i1> [[TMP11]], zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
index 1c62e57edfc46..514d5f974cb16 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
@@ -4,17 +4,13 @@
 define i64 @foo(i32 %tmp7) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[TMP5:%.*]], i32 2
-; CHECK-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 undef, 0
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 undef, i32 0>, i32 [[TMP24]], i32 4
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> [[TMP0]], i32 0, i32 5
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 undef, i32 poison, i32 undef>, i32 [[TMP24]], i32 6
-; CHECK-NEXT:    [[TMP12:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> poison, <4 x i32> [[TMP3]], i64 0)
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP12]], <8 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 poison, i32 2, i32 3, i32 poison, i32 14, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <8 x i32> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 4, i32 5, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison>, i32 [[TMP8:%.*]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP0]], <i32 0, i32 0, i32 poison, i32 0, i32 0, i32 poison, i32 0, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 0>, <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 14, i32 poison, i32 poison, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x i32> [[TMP2]], i32 0, i32 5
+; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw <8 x i32> [[TMP13]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <8 x i32> [[TMP13]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <8 x i32> <i32 8, i32 1, i32 2, i32 3, i32 12, i32 13, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP8:%.*]] = add <8 x i32> zeroinitializer, [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor <8 x i32> [[TMP8]], zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP9]])
@@ -29,7 +25,7 @@ bb:
   %tmp4 = xor i32 %tmp3, 0
   %tmp6 = sub i32 0, 0
   %tmp8 = sub i32 %tmp7, 0
-  %tmp9 = sub nsw i32 0, undef
+  %tmp9 = sub nsw i32 0, poison
   %tmp10 = add nsw i32 0, %tmp6
   %tmp11 = sub nsw i32 0, %tmp8
   %tmp12 = add i32 0, %tmp10
@@ -44,10 +40,10 @@ bb:
   %tmp21 = add i32 %tmp20, %tmp17
   %tmp22 = sub i32 0, 0
   %tmp23 = add i32 0, 0
-  %tmp24 = sub i32 undef, 0
-  %tmp25 = add nsw i32 %tmp23, undef
+  %tmp24 = sub i32 poison, 0
+  %tmp25 = add nsw i32 %tmp23, poison
   %tmp26 = add nsw i32 %tmp24, %tmp22
-  %tmp27 = sub nsw i32 undef, %tmp24
+  %tmp27 = sub nsw i32 poison, %tmp24
   %tmp28 = add i32 0, %tmp25
   %tmp29 = xor i32 %tmp28, 0
   %tmp30 = add i32 0, %tmp26
@@ -58,7 +54,7 @@ bb:
   %tmp35 = add i32 %tmp34, %tmp29
   %tmp36 = add i32 %tmp35, 0
   %tmp37 = add i32 %tmp36, %tmp33
-  %tmp38 = sub nsw i32 0, undef
+  %tmp38 = sub nsw i32 0, poison
   %tmp39 = add i32 0, %tmp38
   %tmp40 = xor i32 %tmp39, 0
   %tmp41 = add i32 0, %tmp37
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
index 02c3173adc654..c6f5308cf54aa 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
@@ -9,9 +9,7 @@ define i32 @foo(ptr nocapture %A, i32 %n, i32 %m) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[N:%.*]], i32 0
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 3, i32 10>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i32> [[SHUFFLE]], <i32 5, i32 9, i32 8, i32 10>
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], splat (i32 9)
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
index b39480b12496b..5a9ea0d292fa0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
@@ -6,26 +6,26 @@
 define i64 @foo() {
 ; CHECK-LABEL: define i64 @foo() {
 ; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[ADD7:%.*]] = add i64 0, 0
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[ADD:%.*]], [[BB3]] ]
-; CHECK-NEXT:    [[PHI2:%.*]] = phi i64 [ [[TMP9:%.*]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i64> [ [[TMP5:%.*]], [[BB3]] ]
 ; CHECK-NEXT:    ret i64 0
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[PHI5:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ 0, [[BB3]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, [[BB]] ], [ [[TMP7:%.*]], [[BB3]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[ADD]] = add i64 [[TMP3]], [[TMP2]]
-; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 0
-; CHECK-NEXT:    [[TMP9]] = or i64 [[PHI5]], 0
-; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP7]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[ADD]], i32 0
+; CHECK-NEXT:    [[PHI4:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ 0, [[BB3]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, [[BB]] ], [ [[TMP3:%.*]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[PHI4]], i32 0
+; CHECK-NEXT:    [[TMP3]] = add <2 x i64> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5]] = add <2 x i64> [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ADD7]]
+; CHECK-NEXT:    [[OR:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[OR]], 0
 ; CHECK-NEXT:    br i1 false, label [[BB3]], label [[BB1:%.*]]
 ;
 ; FORCED-LABEL: define i64 @foo() {
 ; FORCED-NEXT:  bb:
-; FORCED-NEXT:    [[TMP8:%.*]] = add i64 0, 0
+; FORCED-NEXT:    [[ADD7:%.*]] = add i64 0, 0
 ; FORCED-NEXT:    br label [[BB3:%.*]]
 ; FORCED:       bb1:
 ; FORCED-NEXT:    [[TMP0:%.*]] = phi <2 x i64> [ [[TMP5:%.*]], [[BB3]] ]
@@ -36,12 +36,10 @@ define i64 @foo() {
 ; FORCED-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
 ; FORCED-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[PHI5]], i32 0
 ; FORCED-NEXT:    [[TMP7]] = add <2 x i64> [[TMP6]], [[TMP2]]
-; FORCED-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; FORCED-NEXT:    [[TMP4:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
-; FORCED-NEXT:    [[TMP5]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i32> <i32 0, i32 3>
-; FORCED-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[TMP8]]
-; FORCED-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
-; FORCED-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[TMP9]], 0
+; FORCED-NEXT:    [[TMP5]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; FORCED-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[ADD7]]
+; FORCED-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; FORCED-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[TMP8]], 0
 ; FORCED-NEXT:    br i1 false, label [[BB3]], label [[BB1:%.*]]
 ;
 bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll
index 2a5bfa7390770..0198b1c5cb846 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-drop-wrapping-flags.ll
@@ -9,9 +9,7 @@ define i32 @test() {
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i8> poison, i8 [[A_PROMOTED]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i16> [[TMP5]], <i16 0, i16 -1, i16 0, i16 0>
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll
index e6a166c27ac49..230e165e43edc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-extracts-bv-combined.ll
@@ -8,10 +8,8 @@ define i32 @foo() {
 ; CHECK-NEXT:    [[D:%.*]] = load i32, ptr null, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 undef, i32 1, i32 0>, i32 [[D]], i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 2, i32 3, i32 1, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i32> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    store <8 x i32> [[TMP4]], ptr getelementptr inbounds ([64 x i32], ptr null, i64 0, i64 15), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add <8 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    store <8 x i32> [[TMP2]], ptr getelementptr inbounds ([64 x i32], ptr null, i64 0, i64 15), align 4
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
index 1163c8219dabe..034fe82862950 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
@@ -4,6 +4,24 @@
 ; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
 
 define void @foo() {
+; CHECK-LABEL: define void @foo() {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 0, i32 0
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP2]], [[BB4]] ]
+; CHECK-NEXT:    ret void
+;
 ; FORCED-LABEL: define void @foo() {
 ; FORCED-NEXT:  bb:
 ; FORCED-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 0, i32 0
@@ -11,9 +29,7 @@ define void @foo() {
 ; FORCED:       bb1:
 ; FORCED-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
 ; FORCED-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], [[TMP0]]
-; FORCED-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[TMP1]], [[TMP0]]
-; FORCED-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; FORCED-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; FORCED-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
 ; FORCED-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
 ; FORCED-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
 ; FORCED-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
@@ -21,29 +37,9 @@ define void @foo() {
 ; FORCED:       bb4:
 ; FORCED-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
 ; FORCED:       bb5:
-; FORCED-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP4]], [[BB4]] ]
+; FORCED-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP2]], [[BB4]] ]
 ; FORCED-NEXT:    ret void
 ;
-; CHECK-LABEL: define void @foo() {
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    br label [[BB1:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[SHL]], i32 0
-; CHECK-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
-; CHECK-NEXT:    br label [[BB4]]
-; CHECK:       bb4:
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP6]], i32 1
-; CHECK-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
-; CHECK:       bb5:
-; CHECK-NEXT:    [[PHI6:%.*]] = phi i32 [ [[SHL]], [[BB4]] ]
-; CHECK-NEXT:    [[PHI7:%.*]] = phi i32 [ [[TMP8]], [[BB4]] ]
-; CHECK-NEXT:    ret void
-;
 bb:
   br label %bb1
 
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
index cb02f4d10923c..ad8e905a8ca02 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
@@ -330,9 +330,7 @@ define void @only_arcp(ptr %x) {
 define void @addsub_all_nsw(ptr %x) {
 ; CHECK-LABEL: @addsub_all_nsw(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 1, i32 -1, i32 1, i32 -1>
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[X]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -361,9 +359,7 @@ define void @addsub_all_nsw(ptr %x) {
 define void @addsub_some_nsw(ptr %x) {
 ; CHECK-LABEL: @addsub_some_nsw(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 -1, i32 1, i32 -1>
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[X]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -392,9 +388,7 @@ define void @addsub_some_nsw(ptr %x) {
 define void @addsub_no_nsw(ptr %x) {
 ; CHECK-LABEL: @addsub_no_nsw(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], splat (i32 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 -1, i32 1, i32 -1>
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[X]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
index 81f3bf99f3fd8..7fe6941d52da7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-vectorized-in-transform.ll
@@ -9,16 +9,16 @@ define i32 @test(i1 %cond) {
 ; CHECK:       [[BB]]:
 ; CHECK-NEXT:    [[P1:%.*]] = phi i32 [ [[OR92:%.*]], %[[BB]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ [[TMP8:%.*]], %[[BB]] ], [ zeroinitializer, %[[ENTRY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 1, 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>, <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[P1]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> zeroinitializer, [[TMP4]]
 ; CHECK-NEXT:    [[OR92]] = or i32 1, 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[OR92]], i32 0
+; CHECK-NEXT:    [[TMP8]] = xor <2 x i32> [[TMP9]], [[TMP7]]
 ; CHECK-NEXT:    [[OP_RDX:%.*]] = xor i32 [[TMP6]], [[OR92]]
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[OP_RDX]], i32 0
-; CHECK-NEXT:    [[TMP8]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP1]], i32 1
 ; CHECK-NEXT:    br i1 [[COND]], label %[[EXIT:.*]], label %[[BB]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 [[OP_RDX]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
index cda88620ab88a..fff2b72df613e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
@@ -14,10 +14,8 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i16> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub <4 x i16> [[TMP8]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i16> [[TMP9]], <4 x i16> [[TMP10]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT:    [[TMP12:%.*]] = add <4 x i16> zeroinitializer, [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub <4 x i16> zeroinitializer, [[TMP11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i16> [[TMP12]], <4 x i16> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP15:%.*]] = sext <4 x i16> [[TMP14]] to <4 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = sext <4 x i16> [[TMP13]] to <4 x i32>
 ; CHECK-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP2]], align 16
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
index fcc295de62adf..a17ccb4b46ef9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
@@ -6,11 +6,9 @@ define i1 @test() {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[H_PROMOTED118_I_FR:%.*]] = freeze i32 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[H_PROMOTED118_I_FR]], i32 2
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> zeroinitializer, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> zeroinitializer, [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP3]], <4 x i32> <i32 2, i32 2, i32 7, i32 2>
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> zeroinitializer, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> <i32 2, i32 2, i32 7, i32 2>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP5]], <i32 0, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[TMP6]], <i32 1, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
index 6e2a43ac5f9f1..15dd6756cd7db 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
@@ -242,13 +242,18 @@ exit:
 }
 
 define void @store_try_reorder(ptr %dst) {
-; CHECK-LABEL: @store_try_reorder(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 0, 0
-; CHECK-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
-; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @store_try_reorder(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    store <3 x i32> zeroinitializer, ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @store_try_reorder(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[ADD216:%.*]] = sub i32 0, 0
+; POW2-ONLY-NEXT:    [[ARRAYIDX_I1891:%.*]] = getelementptr i32, ptr [[DST]], i64 2
+; POW2-ONLY-NEXT:    store i32 [[ADD216]], ptr [[ARRAYIDX_I1891]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %add = add i32 0, 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
index 869a9d1aee80e..4f3d551e21122 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
@@ -192,9 +192,7 @@ define void @addsub0(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 -2, i32 -3>
-; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <2 x i32> [[TMP2]], <i32 -2, i32 -3>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 -2, i32 3>
 ; CHECK-NEXT:    store <2 x i32> [[TMP5]], ptr [[INCDEC_PTR3]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -225,9 +223,7 @@ define void @addsub1(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 2
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[TMP0]], splat (i32 -1)
-; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw <2 x i32> [[TMP0]], splat (i32 -1)
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP0]], <i32 -1, i32 1>
 ; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll b/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll
index c250029519590..9b6511d0d8284 100644
--- a/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll
+++ b/llvm/test/Transforms/SLPVectorizer/alternate-opcode-sindle-bv.ll
@@ -1,18 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=X86 %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=AARCH64 %}
 
 define <2 x i32> @test(i32 %arg) {
-; CHECK-LABEL: define <2 x i32> @test(
-; CHECK-SAME: i32 [[ARG:%.*]]) {
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[ARG]], 0
-; CHECK-NEXT:    [[MUL:%.*]] = mul i32 0, 1
-; CHECK-NEXT:    [[MUL1:%.*]] = mul i32 [[OR]], [[MUL]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 0, [[MUL1]]
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[OR]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[MUL]], i32 1
-; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+; X86-LABEL: define <2 x i32> @test(
+; X86-SAME: i32 [[ARG:%.*]]) {
+; X86-NEXT:  bb:
+; X86-NEXT:    [[OR:%.*]] = or i32 [[ARG]], 0
+; X86-NEXT:    [[MUL:%.*]] = mul i32 0, 1
+; X86-NEXT:    [[MUL1:%.*]] = mul i32 [[OR]], [[MUL]]
+; X86-NEXT:    [[CMP:%.*]] = icmp ugt i32 0, [[MUL1]]
+; X86-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[OR]], i32 0
+; X86-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[MUL]], i32 1
+; X86-NEXT:    ret <2 x i32> [[TMP1]]
+;
+; AARCH64-LABEL: define <2 x i32> @test(
+; AARCH64-SAME: i32 [[ARG:%.*]]) {
+; AARCH64-NEXT:  bb:
+; AARCH64-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[ARG]], i32 0
+; AARCH64-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[TMP0]], zeroinitializer
+; AARCH64-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; AARCH64-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; AARCH64-NEXT:    [[MUL1:%.*]] = mul i32 [[TMP2]], [[TMP3]]
+; AARCH64-NEXT:    [[CMP:%.*]] = icmp ugt i32 0, [[MUL1]]
+; AARCH64-NEXT:    ret <2 x i32> [[TMP1]]
 ;
 bb:
   %or = or i32 %arg, 0
diff --git a/llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll b/llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll
new file mode 100644
index 0000000000000..c3b0de084b748
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/isOpcodeOrAlt.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S -slp-max-reg-size=1024 %s | FileCheck %s
+
+define void @test(ptr %a, ptr %b) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 0
+; CHECK-NEXT:    [[GEP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[GEP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[TMP0]], <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[GEP4]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %gep0 = getelementptr inbounds i32, ptr %a, i64 0
+  %gep1 = getelementptr inbounds i32, ptr %a, i64 1
+  %gep2 = getelementptr inbounds i32, ptr %a, i64 2
+  %gep3 = getelementptr inbounds i32, ptr %a, i64 3
+  %0 = load i32, ptr %gep0, align 4
+  %1 = load i32, ptr %gep1, align 4
+  %2 = load i32, ptr %gep2, align 4
+  %3 = load i32, ptr %gep3, align 4
+  %op0 = shl i32 %0, 1
+  %op1 = add i32 %1, zeroinitializer
+  %op2 = mul i32 %2, 2
+  %op3 = shl i32 %3, zeroinitializer
+  %gep4 = getelementptr inbounds i32, ptr %b, i64 0
+  %gep5 = getelementptr inbounds i32, ptr %b, i64 1
+  %gep6 = getelementptr inbounds i32, ptr %b, i64 2
+  %gep7 = getelementptr inbounds i32, ptr %b, i64 3
+  store i32 %op0, ptr %gep4, align 4
+  store i32 %op1, ptr %gep5, align 4
+  store i32 %op2, ptr %gep6, align 4
+  store i32 %op3, ptr %gep7, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
index 056b6222cae72..caca410f056c1 100644
--- a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
+++ b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
@@ -6,11 +6,9 @@ define void @func(i32 %0) {
 ; CHECK-SAME: i32 [[TMP0:%.*]]) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 0, i32 0>, i32 [[TMP0]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = or <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i32 [[TMP0]], 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <32 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <32 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP6]] to i64
 ; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP9]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP9]] to i32
diff --git a/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll b/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
index 732b50396a460..cf5927bf58327 100644
--- a/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/shuffle-mask-resized.ll
@@ -12,9 +12,7 @@ define i32 @test() {
 ; CHECK-NEXT:    br i1 false, label [[BB4:%.*]], label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> zeroinitializer, [[TMP2]]
-; CHECK-NEXT:    [[TMP5]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP5]] = add <2 x i32> zeroinitializer, [[TMP2]]
 ; CHECK-NEXT:    br label [[BB1]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi <8 x i32> [ [[TMP1]], [[BB1]] ]
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 6de8165826442..6c4e7cb689b20 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -61,6 +61,7 @@ add_llvm_unittest(SupportTests
   MemoryBufferRefTest.cpp
   MemoryBufferTest.cpp
   MemoryTest.cpp
+  MustacheTest.cpp      
   ModRefTest.cpp
   NativeFormatTests.cpp
   OptimizedStructLayoutTest.cpp
diff --git a/llvm/unittests/Support/MustacheTest.cpp b/llvm/unittests/Support/MustacheTest.cpp
new file mode 100644
index 0000000000000..6ab3d4b01bc1b
--- /dev/null
+++ b/llvm/unittests/Support/MustacheTest.cpp
@@ -0,0 +1,1226 @@
+//===- llvm/unittest/Support/MustacheTest.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Test conforming to Mustache 1.4.2 spec found here:
+// https://github.com/mustache/spec
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Mustache.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::mustache;
+using namespace llvm::json;
+
+TEST(MustacheInterpolation, NoInterpolation) {
+  // Mustache-free templates should render as-is.
+  Value D = {};
+  auto T = Template("Hello from {Mustache}!\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello from {Mustache}!\n", Out);
+}
+
+TEST(MustacheInterpolation, BasicInterpolation) {
+  // Unadorned tags should interpolate content into the template.
+  Value D = Object{{"subject", "World"}};
+  auto T = Template("Hello, {{subject}}!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World!", Out);
+}
+
+TEST(MustacheInterpolation, NoReinterpolation) {
+  // Interpolated tag output should not be re-interpolated.
+  Value D = Object{{"template", "{{planet}}"}, {"planet", "Earth"}};
+  auto T = Template("{{template}}: {{planet}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("{{planet}}: Earth", Out);
+}
+
+TEST(MustacheInterpolation, HTMLEscaping) {
+  // Interpolated tag output should not be re-interpolated.
+  Value D = Object{
+      {"forbidden", "& \" < >"},
+  };
+  auto T = Template("These characters should be HTML escaped: {{forbidden}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("These characters should be HTML escaped: &amp; &quot; &lt; &gt;\n",
+            Out);
+}
+
+TEST(MustacheInterpolation, Ampersand) {
+  // Interpolated tag output should not be re-interpolated.
+  Value D = Object{
+      {"forbidden", "& \" < >"},
+  };
+  auto T =
+      Template("These characters should not be HTML escaped: {{&forbidden}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("These characters should not be HTML escaped: & \" < >\n", Out);
+}
+
+TEST(MustacheInterpolation, BasicIntegerInterpolation) {
+  // Integers should interpolate seamlessly.
+  Value D = Object{{"mph", 85}};
+  auto T = Template("{{mph}} miles an hour!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("85 miles an hour!", Out);
+}
+
+TEST(MustacheInterpolation, AmpersandIntegerInterpolation) {
+  // Integers should interpolate seamlessly.
+  Value D = Object{{"mph", 85}};
+  auto T = Template("{{&mph}} miles an hour!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("85 miles an hour!", Out);
+}
+
+TEST(MustacheInterpolation, BasicDecimalInterpolation) {
+  // Decimals should interpolate seamlessly with proper significance.
+  Value D = Object{{"power", 1.21}};
+  auto T = Template("{{power}} jiggawatts!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1.21 jiggawatts!", Out);
+}
+
+TEST(MustacheInterpolation, BasicNullInterpolation) {
+  // Nulls should interpolate as the empty string.
+  Value D = Object{{"cannot", nullptr}};
+  auto T = Template("I ({{cannot}}) be seen!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("I () be seen!", Out);
+}
+
+TEST(MustacheInterpolation, AmpersandNullInterpolation) {
+  // Nulls should interpolate as the empty string.
+  Value D = Object{{"cannot", nullptr}};
+  auto T = Template("I ({{&cannot}}) be seen!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("I () be seen!", Out);
+}
+
+TEST(MustacheInterpolation, BasicContextMissInterpolation) {
+  // Failed context lookups should default to empty strings.
+  Value D = Object{};
+  auto T = Template("I ({{cannot}}) be seen!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("I () be seen!", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesBasicInterpolation) {
+  // Dotted names should be considered a form of shorthand for sections.
+  Value D = Object{{"person", Object{{"name", "Joe"}}}};
+  auto T = Template("{{person.name}} == {{#person}}{{name}}{{/person}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Joe == Joe", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesAmpersandInterpolation) {
+  // Dotted names should be considered a form of shorthand for sections.
+  Value D = Object{{"person", Object{{"name", "Joe"}}}};
+  auto T = Template("{{&person.name}} == {{#person}}{{&name}}{{/person}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Joe == Joe", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesArbitraryDepth) {
+  // Dotted names should be functional to any level of nesting.
+  Value D = Object{
+      {"a",
+       Object{{"b",
+               Object{{"c",
+                       Object{{"d",
+                               Object{{"e", Object{{"name", "Phil"}}}}}}}}}}}};
+  auto T = Template("{{a.b.c.d.e.name}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Phil", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesBrokenChains) {
+  // Any falsey value prior to the last part of the name should yield ''.
+  Value D = Object{{"a", Object{}}};
+  auto T = Template("{{a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesBrokenChainResolution) {
+  // Each part of a dotted name should resolve only against its parent.
+  Value D =
+      Object{{"a", Object{{"b", Object{}}}}, {"c", Object{{"name", "Jim"}}}};
+  auto T = Template("{{a.b.c.name}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesInitialResolution) {
+  // The first part of a dotted name should resolve as any other name.
+  Value D = Object{
+      {"a",
+       Object{
+           {"b",
+            Object{{"c",
+                    Object{{"d", Object{{"e", Object{{"name", "Phil"}}}}}}}}}}},
+      {"b",
+       Object{{"c", Object{{"d", Object{{"e", Object{{"name", "Wrong"}}}}}}}}}};
+  auto T = Template("{{#a}}{{b.c.d.e.name}}{{/a}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Phil", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesContextPrecedence) {
+  // Dotted names should be resolved against former resolutions.
+  Value D =
+      Object{{"a", Object{{"b", Object{}}}}, {"b", Object{{"c", "ERROR"}}}};
+  auto T = Template("{{#a}}{{b.c}}{{/a}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesAreNotSingleKeys) {
+  // Dotted names shall not be parsed as single, atomic keys
+  Value D = Object{{"a.b", "c"}};
+  auto T = Template("{{a.b}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInterpolation, DottedNamesNoMasking) {
+  // Dotted Names in a given context are unavailable due to dot splitting
+  Value D = Object{{"a.b", "c"}, {"a", Object{{"b", "d"}}}};
+  auto T = Template("{{a.b}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("d", Out);
+}
+
+TEST(MustacheInterpolation, ImplicitIteratorsBasicInterpolation) {
+  // Unadorned tags should interpolate content into the template.
+  Value D = "world";
+  auto T = Template("Hello, {{.}}!\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, world!\n", Out);
+}
+
+TEST(MustacheInterpolation, ImplicitIteratorsAmersand) {
+  // Basic interpolation should be HTML escaped.
+  Value D = "& \" < >";
+  auto T = Template("These characters should not be HTML escaped: {{&.}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("These characters should not be HTML escaped: & \" < >\n", Out);
+}
+
+TEST(MustacheInterpolation, ImplicitIteratorsInteger) {
+  // Integers should interpolate seamlessly.
+  Value D = 85;
+  auto T = Template("{{.}} miles an hour!\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("85 miles an hour!\n", Out);
+}
+
+TEST(MustacheInterpolation, InterpolationSurroundingWhitespace) {
+  // Interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("| {{string}} |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| --- |", Out);
+}
+
+TEST(MustacheInterpolation, AmersandSurroundingWhitespace) {
+  // Interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("| {{&string}} |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| --- |", Out);
+}
+
+TEST(MustacheInterpolation, StandaloneInterpolationWithWhitespace) {
+  // Standalone interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("  {{string}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  ---\n", Out);
+}
+
+TEST(MustacheInterpolation, StandaloneAmpersandWithWhitespace) {
+  // Standalone interpolation should not alter surrounding whitespace.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("  {{&string}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  ---\n", Out);
+}
+
+TEST(MustacheInterpolation, InterpolationWithPadding) {
+  // Superfluous in-tag whitespace should be ignored.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("|{{ string }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|---|", Out);
+}
+
+TEST(MustacheInterpolation, AmpersandWithPadding) {
+  // Superfluous in-tag whitespace should be ignored.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("|{{& string }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|---|", Out);
+}
+
+TEST(MustacheInterpolation, InterpolationWithPaddingAndNewlines) {
+  // Superfluous in-tag whitespace should be ignored.
+  Value D = Object{{"string", "---"}};
+  auto T = Template("|{{ string \n\n\n }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|---|", Out);
+}
+
+TEST(MustacheSections, Truthy) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("{{#boolean}}This should be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("This should be rendered.", Out);
+}
+
+TEST(MustacheSections, Falsey) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("{{#boolean}}This should not be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInterpolation, IsFalseyNull) {
+  // Mustache-free templates should render as-is.
+  Value D = Object{{"boolean", nullptr}};
+  auto T = Template("Hello, {{#boolean}}World{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, ", Out);
+}
+
+TEST(MustacheInterpolation, IsFalseyArray) {
+  // Mustache-free templates should render as-is.
+  Value D = Object{{"boolean", Array()}};
+  auto T = Template("Hello, {{#boolean}}World{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, ", Out);
+}
+
+TEST(MustacheInterpolation, IsFalseyObject) {
+  // Mustache-free templates should render as-is.
+  Value D = Object{{"boolean", Object{}}};
+  auto T = Template("Hello, {{#boolean}}World{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World", Out);
+}
+
+TEST(MustacheInterpolation, DoubleRendering) {
+  // Mustache-free templates should render as-is.
+  Value D1 = Object{{"subject", "World"}};
+  auto T = Template("Hello, {{subject}}!");
+  std::string Out1;
+  raw_string_ostream OS1(Out1);
+  T.render(D1, OS1);
+  EXPECT_EQ("Hello, World!", Out1);
+  std::string Out2;
+  raw_string_ostream OS2(Out2);
+  Value D2 = Object{{"subject", "New World"}};
+  T.render(D2, OS2);
+  EXPECT_EQ("Hello, New World!", Out2);
+}
+
+TEST(MustacheSections, NullIsFalsey) {
+  Value D = Object{{"null", nullptr}};
+  auto T = Template("{{#null}}This should not be rendered.{{/null}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheSections, Context) {
+  Value D = Object{{"context", Object{{"name", "Joe"}}}};
+  auto T = Template("{{#context}}Hi {{name}}.{{/context}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hi Joe.", Out);
+}
+
+TEST(MustacheSections, ParentContexts) {
+  Value D = Object{{"a", "foo"},
+                   {"b", "wrong"},
+                   {"sec", Object{{"b", "bar"}}},
+                   {"c", Object{{"d", "baz"}}}};
+  auto T = Template("{{#sec}}{{a}}, {{b}}, {{c.d}}{{/sec}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("foo, bar, baz", Out);
+}
+
+TEST(MustacheSections, VariableTest) {
+  Value D = Object{{"foo", "bar"}};
+  auto T = Template("{{#foo}}{{.}} is {{foo}}{{/foo}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("bar is bar", Out);
+}
+
+TEST(MustacheSections, ListContexts) {
+  Value D = Object{
+      {"tops",
+       Array{Object{
+           {"tname", Object{{"upper", "A"}, {"lower", "a"}}},
+           {"middles",
+            Array{Object{{"mname", "1"},
+                         {"bottoms", Array{Object{{"bname", "x"}},
+                                           Object{{"bname", "y"}}}}}}}}}}};
+  auto T = Template("{{#tops}}"
+                    "{{#middles}}"
+                    "{{tname.lower}}{{mname}}."
+                    "{{#bottoms}}"
+                    "{{tname.upper}}{{mname}}{{bname}}."
+                    "{{/bottoms}}"
+                    "{{/middles}}"
+                    "{{/tops}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("a1.A1x.A1y.", Out);
+}
+
+TEST(MustacheSections, DeeplyNestedContexts) {
+  Value D = Object{
+      {"a", Object{{"one", 1}}},
+      {"b", Object{{"two", 2}}},
+      {"c", Object{{"three", 3}, {"d", Object{{"four", 4}, {"five", 5}}}}}};
+  auto T = Template(
+      "{{#a}}\n{{one}}\n{{#b}}\n{{one}}{{two}}{{one}}\n{{#c}}\n{{one}}{{two}}{{"
+      "three}}{{two}}{{one}}\n{{#d}}\n{{one}}{{two}}{{three}}{{four}}{{three}}{"
+      "{two}}{{one}}\n{{#five}}\n{{one}}{{two}}{{three}}{{four}}{{five}}{{four}"
+      "}{{three}}{{two}}{{one}}\n{{one}}{{two}}{{three}}{{four}}{{.}}6{{.}}{{"
+      "four}}{{three}}{{two}}{{one}}\n{{one}}{{two}}{{three}}{{four}}{{five}}{{"
+      "four}}{{three}}{{two}}{{one}}\n{{/"
+      "five}}\n{{one}}{{two}}{{three}}{{four}}{{three}}{{two}}{{one}}\n{{/"
+      "d}}\n{{one}}{{two}}{{three}}{{two}}{{one}}\n{{/"
+      "c}}\n{{one}}{{two}}{{one}}\n{{/b}}\n{{one}}\n{{/a}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1\n121\n12321\n1234321\n123454321\n12345654321\n123454321\n1234321"
+            "\n12321\n121\n1\n",
+            Out);
+}
+
+TEST(MustacheSections, List) {
+  Value D = Object{{"list", Array{Object{{"item", 1}}, Object{{"item", 2}},
+                                  Object{{"item", 3}}}}};
+  auto T = Template("{{#list}}{{item}}{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("123", Out);
+}
+
+TEST(MustacheSections, EmptyList) {
+  Value D = Object{{"list", Array{}}};
+  auto T = Template("{{#list}}Yay lists!{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheSections, Doubled) {
+  Value D = Object{{"bool", true}, {"two", "second"}};
+  auto T = Template("{{#bool}}\n* first\n{{/bool}}\n* "
+                    "{{two}}\n{{#bool}}\n* third\n{{/bool}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("* first\n* second\n* third\n", Out);
+}
+
+TEST(MustacheSections, NestedTruthy) {
+  Value D = Object{{"bool", true}};
+  auto T = Template("| A {{#bool}}B {{#bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A B C D E |", Out);
+}
+
+TEST(MustacheSections, NestedFalsey) {
+  Value D = Object{{"bool", false}};
+  auto T = Template("| A {{#bool}}B {{#bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A  E |", Out);
+}
+
+TEST(MustacheSections, ContextMisses) {
+  Value D = Object{};
+  auto T = Template("[{{#missing}}Found key 'missing'!{{/missing}}]");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("[]", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorString) {
+  Value D = Object{{"list", Array{"a", "b", "c", "d", "e"}}};
+  auto T = Template("{{#list}}({{.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(a)(b)(c)(d)(e)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorInteger) {
+  Value D = Object{{"list", Array{1, 2, 3, 4, 5}}};
+  auto T = Template("{{#list}}({{.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(1)(2)(3)(4)(5)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorArray) {
+  Value D = Object{{"list", Array{Array{1, 2, 3}, Array{"a", "b", "c"}}}};
+  auto T = Template("{{#list}}({{#.}}{{.}}{{/.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(123)(abc)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorHTMLEscaping) {
+  Value D = Object{{"list", Array{"&", "\"", "<", ">"}}};
+  auto T = Template("{{#list}}({{.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(&amp;)(&quot;)(&lt;)(&gt;)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorAmpersand) {
+  Value D = Object{{"list", Array{"&", "\"", "<", ">"}}};
+  auto T = Template("{{#list}}({{&.}}){{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(&)(\")(<)(>)", Out);
+}
+
+TEST(MustacheSections, ImplicitIteratorRootLevel) {
+  Value D = Array{Object{{"value", "a"}}, Object{{"value", "b"}}};
+  auto T = Template("{{#.}}({{value}}){{/.}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("(a)(b)", Out);
+}
+
+TEST(MustacheSections, DottedNamesTruthy) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", true}}}}}};
+  auto T = Template("{{#a.b.c}}Here{{/a.b.c}} == Here");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Here == Here", Out);
+}
+
+TEST(MustacheSections, DottedNamesFalsey) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", false}}}}}};
+  auto T = Template("{{#a.b.c}}Here{{/a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheSections, DottedNamesBrokenChains) {
+  Value D = Object{{"a", Object{}}};
+  auto T = Template("{{#a.b.c}}Here{{/a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheSections, SurroundingWhitespace) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template(" | {{#boolean}}\t|\t{{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" | \t|\t | \n", Out);
+}
+
+TEST(MustacheSections, InternalWhitespace) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template(
+      " | {{#boolean}} {{! Important Whitespace }}\n {{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" |  \n  | \n", Out);
+}
+
+TEST(MustacheSections, IndentedInlineSections) {
+  Value D = Object{{"boolean", true}};
+  auto T =
+      Template(" {{#boolean}}YES{{/boolean}}\n {{#boolean}}GOOD{{/boolean}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" YES\n GOOD\n", Out);
+}
+
+TEST(MustacheSections, StandaloneLines) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("| This Is\n{{#boolean}}\n|\n{{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheSections, IndentedStandaloneLines) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("| This Is\n  {{#boolean}}\n|\n  {{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheSections, StandaloneLineEndings) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("|\r\n{{#boolean}}\r\n{{/boolean}}\r\n|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|\r\n|", Out);
+}
+
+TEST(MustacheSections, StandaloneWithoutPreviousLine) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("  {{#boolean}}\n#{{/boolean}}\n/");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("#\n/", Out);
+}
+
+TEST(MustacheSections, StandaloneWithoutNewline) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("#{{#boolean}}\n/\n  {{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("#\n/\n", Out);
+}
+
+TEST(MustacheSections, Padding) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("|{{# boolean }}={{/ boolean }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|=|", Out);
+}
+
+TEST(MustacheInvertedSections, Falsey) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("{{^boolean}}This should be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("This should be rendered.", Out);
+}
+
+TEST(MustacheInvertedSections, Truthy) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("{{^boolean}}This should not be rendered.{{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInvertedSections, NullIsFalsey) {
+  Value D = Object{{"null", nullptr}};
+  auto T = Template("{{^null}}This should be rendered.{{/null}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("This should be rendered.", Out);
+}
+
+TEST(MustacheInvertedSections, Context) {
+  Value D = Object{{"context", Object{{"name", "Joe"}}}};
+  auto T = Template("{{^context}}Hi {{name}}.{{/context}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInvertedSections, List) {
+  Value D = Object{
+      {"list", Array{Object{{"n", 1}}, Object{{"n", 2}}, Object{{"n", 3}}}}};
+  auto T = Template("{{^list}}{{n}}{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustacheInvertedSections, EmptyList) {
+  Value D = Object{{"list", Array{}}};
+  auto T = Template("{{^list}}Yay lists!{{/list}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Yay lists!", Out);
+}
+
+TEST(MustacheInvertedSections, Doubled) {
+  Value D = Object{{"bool", false}, {"two", "second"}};
+  auto T = Template("{{^bool}}\n* first\n{{/bool}}\n* "
+                    "{{two}}\n{{^bool}}\n* third\n{{/bool}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("* first\n* second\n* third\n", Out);
+}
+
+TEST(MustacheInvertedSections, NestedFalsey) {
+  Value D = Object{{"bool", false}};
+  auto T = Template("| A {{^bool}}B {{^bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A B C D E |", Out);
+}
+
+TEST(MustacheInvertedSections, NestedTruthy) {
+  Value D = Object{{"bool", true}};
+  auto T = Template("| A {{^bool}}B {{^bool}}C{{/bool}} D{{/bool}} E |");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| A  E |", Out);
+}
+
+TEST(MustacheInvertedSections, ContextMisses) {
+  Value D = Object{};
+  auto T = Template("[{{^missing}}Cannot find key 'missing'!{{/missing}}]");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("[Cannot find key 'missing'!]", Out);
+}
+
+TEST(MustacheInvertedSections, DottedNamesTruthy) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", true}}}}}};
+  auto T = Template("{{^a.b.c}}Not Here{{/a.b.c}} == ");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" == ", Out);
+}
+
+TEST(MustacheInvertedSections, DottedNamesFalsey) {
+  Value D = Object{{"a", Object{{"b", Object{{"c", false}}}}}};
+  auto T = Template("{{^a.b.c}}Not Here{{/a.b.c}} == Not Here");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Not Here == Not Here", Out);
+}
+
+TEST(MustacheInvertedSections, DottedNamesBrokenChains) {
+  Value D = Object{{"a", Object{}}};
+  auto T = Template("{{^a.b.c}}Not Here{{/a.b.c}} == Not Here");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Not Here == Not Here", Out);
+}
+
+TEST(MustacheInvertedSections, SurroundingWhitespace) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template(" | {{^boolean}}\t|\t{{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" | \t|\t | \n", Out);
+}
+
+TEST(MustacheInvertedSections, InternalWhitespace) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template(
+      " | {{^boolean}} {{! Important Whitespace }}\n {{/boolean}} | \n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" |  \n  | \n", Out);
+}
+
+TEST(MustacheInvertedSections, IndentedInlineSections) {
+  Value D = Object{{"boolean", false}};
+  auto T =
+      Template(" {{^boolean}}NO{{/boolean}}\n {{^boolean}}WAY{{/boolean}}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ(" NO\n WAY\n", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneLines) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("| This Is\n{{^boolean}}\n|\n{{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneIndentedLines) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("| This Is\n  {{^boolean}}\n|\n  {{/boolean}}\n| A Line\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| This Is\n|\n| A Line\n", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneLineEndings) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("|\r\n{{^boolean}}\r\n{{/boolean}}\r\n|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|\r\n|", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneWithoutPreviousLine) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("  {{^boolean}}\n^{{/boolean}}\n/");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("^\n/", Out);
+}
+
+TEST(MustacheInvertedSections, StandaloneWithoutNewline) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("^{{^boolean}}\n/\n  {{/boolean}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("^\n/\n", Out);
+}
+
+TEST(MustacheInvertedSections, Padding) {
+  Value D = Object{{"boolean", false}};
+  auto T = Template("|{{^ boolean }}={{/ boolean }}|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|=|", Out);
+}
+
+TEST(MustachePartials, BasicBehavior) {
+  Value D = Object{};
+  auto T = Template("{{>text}}");
+  T.registerPartial("text", "from partial");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("from partial", Out);
+}
+
+TEST(MustachePartials, FailedLookup) {
+  Value D = Object{};
+  auto T = Template("{{>text}}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("", Out);
+}
+
+TEST(MustachePartials, Context) {
+  Value D = Object{{"text", "content"}};
+  auto T = Template("{{>partial}}");
+  T.registerPartial("partial", "*{{text}}*");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("*content*", Out);
+}
+
+TEST(MustachePartials, Recursion) {
+  Value D =
+      Object{{"content", "X"},
+             {"nodes", Array{Object{{"content", "Y"}, {"nodes", Array{}}}}}};
+  auto T = Template("{{>node}}");
+  T.registerPartial("node", "{{content}}({{#nodes}}{{>node}}{{/nodes}})");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("X(Y())", Out);
+}
+
+TEST(MustachePartials, Nested) {
+  Value D = Object{{"a", "hello"}, {"b", "world"}};
+  auto T = Template("{{>outer}}");
+  T.registerPartial("outer", "*{{a}} {{>inner}}*");
+  T.registerPartial("inner", "{{b}}!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("*hello world!*", Out);
+}
+
+TEST(MustachePartials, SurroundingWhitespace) {
+  Value D = Object{};
+  auto T = Template("| {{>partial}} |");
+  T.registerPartial("partial", "\t|\t");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("| \t|\t |", Out);
+}
+
+TEST(MustachePartials, InlineIndentation) {
+  Value D = Object{{"data", "|"}};
+  auto T = Template("  {{data}}  {{> partial}}\n");
+  T.registerPartial("partial", "<\n<");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  |  <\n<\n", Out);
+}
+
+TEST(MustachePartials, PaddingWhitespace) {
+  Value D = Object{{"boolean", true}};
+  auto T = Template("|{{> partial }}|");
+  T.registerPartial("partial", "[]");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|[]|", Out);
+}
+
+TEST(MustacheLambdas, BasicInterpolation) {
+  Value D = Object{};
+  auto T = Template("Hello, {{lambda}}!");
+  Lambda L = []() -> llvm::json::Value { return "World"; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World!", Out);
+}
+
+TEST(MustacheLambdas, InterpolationExpansion) {
+  Value D = Object{{"planet", "World"}};
+  auto T = Template("Hello, {{lambda}}!");
+  Lambda L = []() -> llvm::json::Value { return "{{planet}}"; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Hello, World!", Out);
+}
+
+TEST(MustacheLambdas, BasicMultipleCalls) {
+  Value D = Object{};
+  auto T = Template("{{lambda}} == {{lambda}} == {{lambda}}");
+  int I = 0;
+  Lambda L = [&I]() -> llvm::json::Value {
+    I += 1;
+    return I;
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1 == 2 == 3", Out);
+}
+
+TEST(MustacheLambdas, Escaping) {
+  Value D = Object{};
+  auto T = Template("<{{lambda}}{{&lambda}}");
+  Lambda L = []() -> llvm::json::Value { return ">"; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<&gt;>", Out);
+}
+
+TEST(MustacheLambdas, Sections) {
+  Value D = Object{};
+  auto T = Template("<{{#lambda}}{{x}}{{/lambda}}>");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value {
+    if (Text == "{{x}}") {
+      return "yes";
+    }
+    return "no";
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<yes>", Out);
+}
+
+TEST(MustacheLambdas, SectionExpansion) {
+  Value D = Object{
+      {"planet", "Earth"},
+  };
+  auto T = Template("<{{#lambda}}-{{/lambda}}>");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value {
+    SmallString<128> Result;
+    Result += Text;
+    Result += "{{planet}}";
+    Result += Text;
+    return Result;
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<-Earth->", Out);
+}
+
+TEST(MustacheLambdas, SectionsMultipleCalls) {
+  Value D = Object{};
+  auto T = Template("{{#lambda}}FILE{{/lambda}} != {{#lambda}}LINE{{/lambda}}");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value {
+    SmallString<128> Result;
+    Result += "__";
+    Result += Text;
+    Result += "__";
+    return Result;
+  };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("__FILE__ != __LINE__", Out);
+}
+
+TEST(MustacheLambdas, InvertedSections) {
+  Value D = Object{{"static", "static"}};
+  auto T = Template("<{{^lambda}}{{static}}{{/lambda}}>");
+  SectionLambda L = [](StringRef Text) -> llvm::json::Value { return false; };
+  T.registerLambda("lambda", L);
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("<>", Out);
+}
+
+TEST(MustacheComments, Inline) {
+  // Comment blocks should be removed from the template.
+  Value D = {};
+  auto T = Template("12345{{! Comment Block! }}67890");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1234567890", Out);
+}
+
+TEST(MustacheComments, Multiline) {
+  // Multiline comments should be permitted.
+  Value D = {};
+  auto T =
+      Template("12345{{!\n  This is a\n  multi-line comment...\n}}67890\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("1234567890\n", Out);
+}
+
+TEST(MustacheComments, Standalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T = Template("Begin.\n{{! Comment Block! }}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, IndentedStandalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T = Template("Begin.\n  {{! Indented Comment Block! }}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, StandaloneLineEndings) {
+  // "\r\n" should be considered a newline for standalone tags.
+  Value D = {};
+  auto T = Template("|\r\n{{! Standalone Comment }}\r\n|");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("|\r\n|", Out);
+}
+
+TEST(MustacheComments, StandaloneWithoutPreviousLine) {
+  // Standalone tags should not require a newline to precede them.
+  Value D = {};
+  auto T = Template("  {{! I'm Still Standalone }}\n!");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("!", Out);
+}
+
+TEST(MustacheComments, StandaloneWithoutNewline) {
+  // Standalone tags should not require a newline to follow them.
+  Value D = {};
+  auto T = Template("!\n  {{! I'm Still Standalone }}");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("!\n", Out);
+}
+
+TEST(MustacheComments, MultilineStandalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T = Template("Begin.\n{{!\nSomething's going on here...\n}}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, IndentedMultilineStandalone) {
+  // All standalone comment lines should be removed.
+  Value D = {};
+  auto T =
+      Template("Begin.\n  {{!\n    Something's going on here...\n  }}\nEnd.\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("Begin.\nEnd.\n", Out);
+}
+
+TEST(MustacheComments, IndentedInline) {
+  // Inline comments should not strip whitespace.
+  Value D = {};
+  auto T = Template("  12 {{! 34 }}\n");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("  12 \n", Out);
+}
+
+TEST(MustacheComments, SurroundingWhitespace) {
+  // Comment removal should preserve surrounding whitespace.
+  Value D = {};
+  auto T = Template("12345 {{! Comment Block! }} 67890");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("12345  67890", Out);
+}
+
+TEST(MustacheComments, VariableNameCollision) {
+  // Comments must never render, even if a variable with the same name exists.
+  Value D = Object{
+      {"! comment", 1}, {"! comment ", 2}, {"!comment", 3}, {"comment", 4}};
+  auto T = Template("comments never show: >{{! comment }}<");
+  std::string Out;
+  raw_string_ostream OS(Out);
+  T.render(D, OS);
+  EXPECT_EQ("comments never show: ><", Out);
+}
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index c398b6f9bf7cd..8d2fc99f84670 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1245,7 +1245,7 @@ void SubtargetEmitter::genSchedClassTables(const CodeGenProcModel &ProcModel,
             PrintFatalError(
                 WriteRes->getLoc(),
                 Twine("Inconsistent resource cycles: AcquireAtCycles "
-                      "< ReleaseAtCycles must hold."));
+                      "<= ReleaseAtCycles must hold."));
           }
           if (AcquireAtCycles[PRIdx] < 0) {
             PrintFatalError(WriteRes->getLoc(),
diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
index 73d3326ba5fab..2d3277fcb6c13 100644
--- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -43,6 +43,7 @@ static_library("Sema") {
     "CodeCompleteConsumer.cpp",
     "DeclSpec.cpp",
     "DelayedDiagnostic.cpp",
+    "HLSLBuiltinTypeDeclBuilder.cpp",
     "HLSLExternalSemaSource.cpp",
     "HeuristicResolver.cpp",
     "IdentifierResolver.cpp",
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index 14e15173de7bc..bd01c5f704b6b 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -1882,7 +1882,7 @@ def Tosa_ConcatOp : Tosa_InferTensorTypeOp<"concat"> {
 
   list<Availability> availability = [
     Profile<[Tosa_PRO_INT, Tosa_PRO_FP]>,
-    Extension<[Tosa_EXT_FP8E4M3, Tosa_EXT_FP8E5M2, Tosa_EXT_BF16]>,
+    Extension<[Tosa_EXT_FP8E4M3, Tosa_EXT_FP8E5M2, Tosa_EXT_BF16, Tosa_EXT_INT16]>,
   ];
 
   let hasCanonicalizer = 1;
@@ -2318,7 +2318,7 @@ def Tosa_CastOp: Tosa_Op<"cast", [Pure,
 
   list<Availability> availability = [
     Profile<[Tosa_PRO_INT, Tosa_PRO_FP]>,
-    Extension<[Tosa_EXT_INT16, Tosa_EXT_FP8E4M3, Tosa_EXT_FP8E5M2, Tosa_EXT_BF16]>,
+    Extension<[Tosa_EXT_FP8E4M3, Tosa_EXT_FP8E5M2, Tosa_EXT_BF16]>,
   ];
 
   let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)";
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaShapeOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaShapeOps.td
index d97ba75231aa8..4eed6cb62e151 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaShapeOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaShapeOps.td
@@ -75,6 +75,11 @@ def Tosa_ConstShapeOp : Tosa_ShapeOp<"const_shape", [ConstantLike, Pure]> {
 
   let results = (outs Tosa_Shape : $output);
 
+  list<Availability> availability = [
+    Profile<[Tosa_PRO_INT, Tosa_PRO_FP]>,
+    Extension<[]>,
+  ];
+
   let hasVerifier = 1;
 }
 
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp
index 9a45d83c61737..9c6658c9a5bf8 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/DialectResourceBlobManager.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/ADT/APFloat.h"
@@ -176,6 +177,28 @@ DenseElementsAttr transposeType(const RangeType &data, ShapedType inputType,
                                 llvm::ArrayRef<ElementType>(outputValues));
 }
 
+// Try to get the values of a DenseResourceElementsAttr construct
+template <typename T>
+std::optional<ArrayRef<T>> tryGetDenseResourceValues(ElementsAttr attr) {
+  if (auto denseResource = dyn_cast<DenseResourceElementsAttr>(attr)) {
+    // Check that the resource memory blob exists
+    AsmResourceBlob *blob = denseResource.getRawHandle().getBlob();
+    if (!blob)
+      return std::nullopt;
+
+    // Check that the data are in a valid form
+    bool isSplat = false;
+    if (!DenseElementsAttr::isValidRawBuffer(attr.getShapedType(),
+                                             blob->getData(), isSplat)) {
+      return std::nullopt;
+    }
+
+    return blob->template getDataAs<T>();
+  }
+
+  return std::nullopt;
+}
+
 // A type specialized transposition of an ElementsAttr.
 // This implementation tries to operate on the underlying data in its raw
 // representation when possible to avoid allocating a large number of Attribute
@@ -183,6 +206,7 @@ DenseElementsAttr transposeType(const RangeType &data, ShapedType inputType,
 DenseElementsAttr transpose(ElementsAttr attr, ShapedType inputType,
                             ShapedType outputType,
                             llvm::ArrayRef<int64_t> permValues) {
+  // Handle generic ElementsAttr
   if (auto data = attr.tryGetValues<bool>())
     return transposeType(*data, inputType, outputType, permValues);
 
@@ -204,6 +228,35 @@ DenseElementsAttr transpose(ElementsAttr attr, ShapedType inputType,
   if (auto data = attr.tryGetValues<APFloat>())
     return transposeType(*data, inputType, outputType, permValues);
 
+  // Handle DenseResourceElementsAttr
+  if (isa<DenseResourceElementsAttr>(attr)) {
+    auto elementTy = attr.getElementType();
+
+    if (auto data = tryGetDenseResourceValues<bool>(attr);
+        data && elementTy.isInteger(1))
+      return transposeType(*data, inputType, outputType, permValues);
+
+    if (auto data = tryGetDenseResourceValues<int8_t>(attr);
+        data && elementTy.isInteger(8))
+      return transposeType(*data, inputType, outputType, permValues);
+
+    if (auto data = tryGetDenseResourceValues<int16_t>(attr);
+        data && elementTy.isInteger(16))
+      return transposeType(*data, inputType, outputType, permValues);
+
+    if (auto data = tryGetDenseResourceValues<int32_t>(attr);
+        data && elementTy.isInteger(32))
+      return transposeType(*data, inputType, outputType, permValues);
+
+    if (auto data = tryGetDenseResourceValues<int64_t>(attr);
+        data && elementTy.isInteger(64))
+      return transposeType(*data, inputType, outputType, permValues);
+
+    if (auto data = tryGetDenseResourceValues<float>(attr);
+        data && elementTy.isF32())
+      return transposeType(*data, inputType, outputType, permValues);
+  }
+
   return nullptr;
 }
 
diff --git a/mlir/test/Dialect/Tosa/availability.mlir b/mlir/test/Dialect/Tosa/availability.mlir
index ff910a40cf219..4e332bc6c1c43 100644
--- a/mlir/test/Dialect/Tosa/availability.mlir
+++ b/mlir/test/Dialect/Tosa/availability.mlir
@@ -507,7 +507,7 @@ func.func @test_reduce_sum(%arg0: tensor<13x21x3xf32>) -> tensor<1x21x3xf32> {
 // CHECK-LABEL: concat
 func.func @test_concat(%arg0: tensor<13x21x3xf32>, %arg1: tensor<13x21x3xf32>) -> tensor<26x21x3xf32> {
   // CHECK: profiles: [ [pro_int, pro_fp] ]
-  // CHECK: extensions: [ [fp8e4m3, fp8e5m2, bf16] ]
+  // CHECK: extensions: [ [fp8e4m3, fp8e5m2, bf16, int16] ]
   %0 = tosa.concat %arg0, %arg1 {axis = 0 : i32} : (tensor<13x21x3xf32>, tensor<13x21x3xf32>) -> tensor<26x21x3xf32>
   return %0 : tensor<26x21x3xf32>
 }
@@ -606,7 +606,7 @@ func.func @test_resize(%arg0: tensor<1x32x32x8xf32>) -> tensor<1x64x64x8xf32> {
 // CHECK-LABEL: cast
 func.func @test_cast1(%arg0: tensor<13x21x3xi32>) -> tensor<13x21x3xf32> {
   // CHECK: profiles: [ [pro_int, pro_fp] ]
-  // CHECK: extensions: [ [int16, fp8e4m3, fp8e5m2, bf16] ]
+  // CHECK: extensions: [ [fp8e4m3, fp8e5m2, bf16] ]
   %0 = tosa.cast %arg0 : (tensor<13x21x3xi32>) -> tensor<13x21x3xf32>
   return %0 : tensor<13x21x3xf32>
 }
diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
index 8ac1e177ae4d4..3f7de9357297e 100644
--- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir
+++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
@@ -108,18 +108,18 @@ func.func @transpose_nofold_quantized_types() -> tensor<1x1x2x2x!quant.uniform<i
   return %0: tensor<1x1x2x2x!quant.uniform<i8<-127:127>:f32:3, {1.000000e-01,1.000000e-01}>>
 }
 
-// CHECK-LABEL: @transpose_nofold_dense_resource
-func.func @transpose_nofold_dense_resource() -> tensor<2x2xf32> {
+// CHECK-LABEL: @transpose_fold_dense_resource
+func.func @transpose_fold_dense_resource() -> tensor<2x2xf32> {
   %0 = "tosa.const"() <{values = dense_resource<resource> : tensor<2x2xf32>}> : () -> tensor<2x2xf32>
 
-  // CHECK: tosa.transpose
+  // CHECK-NOT: tosa.transpose
   %2 = tosa.transpose %0 { perms = array<i32: 1, 0> }: (tensor<2x2xf32>) -> tensor<2x2xf32>
   return %2 : tensor<2x2xf32>
 }
 {-#
   dialect_resources: {
     builtin: {
-      resource: "0x08000000010000000000000002000000000000000300000000000000"
+      resource: "0x040000003f800000400000004040000040800000"
     }
   }
 #-}
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index e83d38a14f77f..b2ede888b542d 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -576,8 +576,7 @@ struct AMDGPUKernelTy : public GenericKernelTy {
   /// Get the HSA kernel object representing the kernel function.
   uint64_t getKernelObject() const { return KernelObject; }
 
-  /// Get the size of implicitargs based on the code object version
-  /// @return 56 for cov4 and 256 for cov5
+  /// Get the size of implicitargs based on the code object version.
   uint32_t getImplicitArgsSize() const { return ImplicitArgsSize; }
 
   /// Indicates whether or not we need to set up our own private segment size.
@@ -3386,20 +3385,17 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   if (auto Err = AMDGPUDevice.getStream(AsyncInfoWrapper, Stream))
     return Err;
 
-  // Only COV5 implicitargs needs to be set. COV4 implicitargs are not used.
-  if (ImplArgs &&
-      getImplicitArgsSize() == sizeof(hsa_utils::AMDGPUImplicitArgsTy)) {
-    ImplArgs->BlockCountX = NumBlocks[0];
-    ImplArgs->BlockCountY = NumBlocks[1];
-    ImplArgs->BlockCountZ = NumBlocks[2];
-    ImplArgs->GroupSizeX = NumThreads[0];
-    ImplArgs->GroupSizeY = NumThreads[1];
-    ImplArgs->GroupSizeZ = NumThreads[2];
-    ImplArgs->GridDims = NumBlocks[2] * NumThreads[2] > 1
-                             ? 3
-                             : 1 + (NumBlocks[1] * NumThreads[1] != 1);
-    ImplArgs->DynamicLdsSize = KernelArgs.DynCGroupMem;
-  }
+  // Set the COV5+ implicit arguments to the appropriate values.
+  ImplArgs->BlockCountX = NumBlocks[0];
+  ImplArgs->BlockCountY = NumBlocks[1];
+  ImplArgs->BlockCountZ = NumBlocks[2];
+  ImplArgs->GroupSizeX = NumThreads[0];
+  ImplArgs->GroupSizeY = NumThreads[1];
+  ImplArgs->GroupSizeZ = NumThreads[2];
+  ImplArgs->GridDims = NumBlocks[2] * NumThreads[2] > 1
+                           ? 3
+                           : 1 + (NumBlocks[1] * NumThreads[1] != 1);
+  ImplArgs->DynamicLdsSize = KernelArgs.DynCGroupMem;
 
   // Push the kernel launch into the stream.
   return Stream->pushKernelLaunch(*this, AllArgs, NumThreads, NumBlocks,
diff --git a/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h b/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h
index 43be4e8edeba4..609ead942dbb3 100644
--- a/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h
+++ b/offload/plugins-nextgen/amdgpu/utils/UtilitiesRTL.h
@@ -40,17 +40,10 @@ struct AMDGPUImplicitArgsTy {
   uint8_t Unused2[132]; // 132 byte offset.
 };
 
-// Dummy struct for COV4 implicitargs.
-struct AMDGPUImplicitArgsTyCOV4 {
-  uint8_t Unused[56];
-};
-
 /// Returns the size in bytes of the implicit arguments of AMDGPU kernels.
 /// `Version` is the ELF ABI version, e.g. COV5.
 inline uint32_t getImplicitArgsSize(uint16_t Version) {
-  return Version < ELF::ELFABIVERSION_AMDGPU_HSA_V5
-             ? sizeof(AMDGPUImplicitArgsTyCOV4)
-             : sizeof(AMDGPUImplicitArgsTy);
+  return sizeof(AMDGPUImplicitArgsTy);
 }
 
 /// Reads the AMDGPU specific metadata from the ELF file and propagates the
diff --git a/offload/plugins-nextgen/common/src/Utils/ELF.cpp b/offload/plugins-nextgen/common/src/Utils/ELF.cpp
index 44d1c737e2efb..b33101b99aa10 100644
--- a/offload/plugins-nextgen/common/src/Utils/ELF.cpp
+++ b/offload/plugins-nextgen/common/src/Utils/ELF.cpp
@@ -65,10 +65,9 @@ checkMachineImpl(const object::ELFObjectFile<ELFT> &ELFObj, uint16_t EMachine) {
   if (Header.e_machine == EM_AMDGPU) {
     if (Header.e_ident[EI_OSABI] != ELFOSABI_AMDGPU_HSA)
       return createError("Invalid AMD OS/ABI, must be AMDGPU_HSA");
-    if (Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V4 &&
-        Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V5 &&
+    if (Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V5 &&
         Header.e_ident[EI_ABIVERSION] != ELFABIVERSION_AMDGPU_HSA_V6)
-      return createError("Invalid AMD ABI version, must be version 4 or above");
+      return createError("Invalid AMD ABI version, must be version 5 or above");
     if ((Header.e_flags & EF_AMDGPU_MACH) < EF_AMDGPU_MACH_AMDGCN_GFX700 ||
         (Header.e_flags & EF_AMDGPU_MACH) >
             EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
index 7fb50403682a7..63ddebdadbdc9 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
@@ -2,7 +2,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# Tests for LLVM libc string.h functions.
+# Tests for LLVM libc sys/epoll.h functions.
 
 load("//libc/test:libc_test_rules.bzl", "libc_test")
 
@@ -17,6 +17,9 @@ libc_test(
         "//libc:epoll_create",
         "//libc:close",
     ],
+    deps = [
+        "//libc/test/UnitTest:errno_test_helpers",
+    ],
 )
 
 libc_test(
@@ -28,6 +31,7 @@ libc_test(
     ],
     deps = [
         "//libc:hdr_sys_epoll_macros",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
 
@@ -43,6 +47,7 @@ libc_test(
     deps = [
         "//libc:hdr_sys_epoll_macros",
         "//libc:types_struct_epoll_event",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
 
@@ -59,6 +64,7 @@ libc_test(
     deps = [
         "//libc:hdr_sys_epoll_macros",
         "//libc:types_struct_epoll_event",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
 
@@ -75,6 +81,7 @@ libc_test(
     deps = [
         "//libc:hdr_sys_epoll_macros",
         "//libc:types_struct_epoll_event",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )
 
@@ -92,5 +99,6 @@ libc_test(
         "//libc:hdr_sys_epoll_macros",
         "//libc:types_struct_epoll_event",
         "//libc:types_struct_timespec",
+        "//libc/test/UnitTest:errno_test_helpers",
     ],
 )

From 4eac4350b8a540aba5c5ed658353b4f6abaf329b Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <pcc@google.com>
Date: Thu, 5 Jun 2025 22:15:52 -0700
Subject: [PATCH 2/3] Address comments

Created using spr 1.3.6-beta.1
---
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp               | 6 ++++--
 llvm/lib/Target/AArch64/AArch64InstrInfo.td                 | 3 +--
 .../AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll     | 4 ++--
 llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll  | 4 ++--
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 80924c4858ccc..716cae5395ebc 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -173,7 +173,7 @@ class AArch64AsmPrinter : public AsmPrinter {
                              const MachineOperand *AUTAddrDisc,
                              Register Scratch,
                              std::optional<AArch64PACKey::ID> PACKey,
-                             uint64_t PACDisc, unsigned PACAddrDisc);
+                             uint64_t PACDisc, Register PACAddrDisc);
 
   // Emit the sequence to compute the discriminator.
   //
@@ -1871,6 +1871,8 @@ Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
                                                      Register AddrDisc,
                                                      Register ScratchReg,
                                                      bool MayUseAddrAsScratch) {
+  assert(ScratchReg == AArch64::X16 || ScratchReg == AArch64::X17 ||
+         !STI->isX16X17Safer());
   // So far we've used NoRegister in pseudos.  Now we need real encodings.
   if (AddrDisc == AArch64::NoRegister)
     AddrDisc = AArch64::XZR;
@@ -2073,7 +2075,7 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(
     Register AUTVal, AArch64PACKey::ID AUTKey, uint64_t AUTDisc,
     const MachineOperand *AUTAddrDisc, Register Scratch,
     std::optional<AArch64PACKey::ID> PACKey, uint64_t PACDisc,
-    unsigned PACAddrDisc) {
+    Register PACAddrDisc) {
   const bool IsAUTPAC = PACKey.has_value();
 
   // We expand AUT/AUTPAC into a sequence of the form
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 37acdc3e9f7cc..caa247c3c2ce4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1955,8 +1955,7 @@ let Predicates = [HasPAuth] in {
   def AUTxMxN : Pseudo<(outs GPR64:$AuthVal, GPR64common:$Scratch),
                        (ins GPR64:$Val, i32imm:$Key,
                             i64imm:$Disc, GPR64:$AddrDisc),
-                       []>, Sched<[WriteI, ReadI]> {
-    let Constraints = "$AuthVal = $Val";
+                       [], "$AuthVal = $Val">, Sched<[WriteI, ReadI]> {
     let isCodeGenOnly = 1;
     let hasSideEffects = 0;
     let mayStore = 0;
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll
index e6651c83af8af..e2aea6df78250 100644
--- a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll
+++ b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll
@@ -165,10 +165,10 @@ define i64 @test_resign_blend_and_const(i64 %arg, i64 %arg1) {
 ; CHECKED-NEXT:    mov x17, x16
 ; CHECKED-NEXT:    xpacd x17
 ; CHECKED-NEXT:    cmp x16, x17
-; CHECKED-NEXT:    b.eq [[L]]auth_success_[[N2:[0-9]+]]
+; CHECKED-NEXT:    b.eq [[L]]auth_success_1
 ; CHECKED-NEXT:    mov x16, x17
 ; CHECKED-NEXT:    b [[L]]resign_end_1
-; CHECKED-NEXT:  Lauth_success_[[N2]]:
+; CHECKED-NEXT:  Lauth_success_1:
 ; CHECKED-NEXT:    mov x17, #56789
 ; CHECKED-NEXT:    pacdb x16, x17
 ; CHECKED-NEXT:  Lresign_end_1:
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll
index 47fa3f8e88c41..f95dc5600aec8 100644
--- a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll
+++ b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll
@@ -687,10 +687,10 @@ define i64 @test_resign_da_constdisc(i64 %arg, i64 %arg1) {
 ; CHECKED-NEXT:    mov x17, x16
 ; CHECKED-NEXT:    xpacd x17
 ; CHECKED-NEXT:    cmp x16, x17
-; CHECKED-NEXT:    b.eq [[L]]auth_success_[[N1:[0-9]+]]
+; CHECKED-NEXT:    b.eq [[L]]auth_success_7
 ; CHECKED-NEXT:    mov x16, x17
 ; CHECKED-NEXT:    b [[L]]resign_end_6
-; CHECKED-NEXT:  Lauth_success_[[N1]]:
+; CHECKED-NEXT:  Lauth_success_7:
 ; CHECKED-NEXT:    mov x17, #256
 ; CHECKED-NEXT:    pacda x16, x17
 ; CHECKED-NEXT:  Lresign_end_6:

From 01161f76e5fbc64b96300171a5c09962fa9046f8 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <pcc@google.com>
Date: Wed, 9 Jul 2025 13:46:14 -0700
Subject: [PATCH 3/3] Address comment

Created using spr 1.3.6-beta.1
---
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 4befc321acb3c..3d740a0cb3dc7 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -178,14 +178,13 @@ class AArch64AsmPrinter : public AsmPrinter {
 
   // Emit the sequence to compute the discriminator.
   //
-  // ScratchReg should be x16/x17.
-  //
-  // The returned register is either unmodified AddrDisc or x16/x17.
+  // The returned register is either unmodified AddrDisc or ScratchReg.
   //
   // If the expanded pseudo is allowed to clobber AddrDisc register, setting
   // MayUseAddrAsScratch may save one MOV instruction, provided the address
   // is already in x16/x17 (i.e. return x16/x17 which is the *modified* AddrDisc
-  // register at the same time):
+  // register at the same time) or the OS doesn't make it safer to use x16/x17
+  // (see AArch64Subtarget::isX16X17Safer()):
   //
   //   mov   x17, x16
   //   movk  x17, #1234, lsl #48