diff --git a/bolt/test/X86/encoding-validation.s b/bolt/test/X86/encoding-validation.s
index 0c716abc4cc0cf4..c01361296489502 100644
--- a/bolt/test/X86/encoding-validation.s
+++ b/bolt/test/X86/encoding-validation.s
@@ -2,7 +2,7 @@
 
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux %s -o %t.o
 # RUN: ld.lld %t.o -o %t.exe -q
-# RUN: llvm-bolt %t.exe --relocs -o %t.out --check-encoding |& FileCheck %s
+# RUN: llvm-bolt %t.exe --relocs -o %t.out --check-encoding 2>&1 | FileCheck %s
 
   .text
   .globl _start
diff --git a/bolt/test/X86/gotpcrelx.s b/bolt/test/X86/gotpcrelx.s
index 6dec125c6a72bdf..c5bc7ac7d0d4723 100644
--- a/bolt/test/X86/gotpcrelx.s
+++ b/bolt/test/X86/gotpcrelx.s
@@ -10,11 +10,11 @@
 # RUN: ld.lld %t.o -o %t.pie.exe -q -pie
 # RUN: ld.lld %t.o -o %t.no-relax.exe -q --no-relax
 # RUN: llvm-bolt %t.exe --relocs -o %t.out --print-cfg --print-only=_start \
-# RUN:   |& FileCheck --check-prefix=BOLT %s
+# RUN:   2>&1 | FileCheck --check-prefix=BOLT %s
 # RUN: llvm-bolt %t.pie.exe -o %t.null --print-cfg --print-only=_start \
-# RUN:   |& FileCheck --check-prefix=PIE-BOLT %s
+# RUN:   2>&1 | FileCheck --check-prefix=PIE-BOLT %s
 # RUN: llvm-bolt %t.no-relax.exe -o %t.null --print-cfg --print-only=_start \
-# RUN:   |& FileCheck --check-prefix=NO-RELAX-BOLT %s
+# RUN:   2>&1 | FileCheck --check-prefix=NO-RELAX-BOLT %s
 # RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex \
 # RUN:   %t.out | FileCheck --check-prefix=DISASM %s
 
diff --git a/bolt/test/X86/icf-jump-tables.test b/bolt/test/X86/icf-jump-tables.test
index 41820008b01338b..57137b79bc317e1 100644
--- a/bolt/test/X86/icf-jump-tables.test
+++ b/bolt/test/X86/icf-jump-tables.test
@@ -4,7 +4,7 @@
 # REQUIRES: system-linux
 
 # RUN: %clang %cflags -O1 -g %p/../Inputs/icf-jump-tables.c -o %t.exe -Wl,-q
-# RUN: llvm-bolt %t.exe --icf -o %t.bolt |& FileCheck %s
+# RUN: llvm-bolt %t.exe --icf -o %t.bolt 2>&1 | FileCheck %s
 
 ## Check that BOLT successfully folded a function with jump table:
 # CHECK: ICF folded {{.*}}. {{[^0]}} functions had jump tables.
diff --git a/bolt/test/X86/indirect-goto-pie.test b/bolt/test/X86/indirect-goto-pie.test
index 81cff9a32fbbddc..3311c1aec061c5e 100644
--- a/bolt/test/X86/indirect-goto-pie.test
+++ b/bolt/test/X86/indirect-goto-pie.test
@@ -6,7 +6,7 @@ REQUIRES: x86_64-linux
 
 RUN: %clang %S/Inputs/indirect_goto.c -o %t -fpic -pie -Wl,-q
 RUN: not llvm-bolt %t -o %t.bolt --relocs=1 --print-cfg --print-only=main \
-RUN:   |& FileCheck %s
+RUN:   2>&1 | FileCheck %s
 
 ## Check that processing works if main() is skipped.
 RUN: llvm-bolt %t -o %t.bolt --relocs=1 --skip-funcs=main
diff --git a/bolt/test/X86/jump-table-func-entry.s b/bolt/test/X86/jump-table-func-entry.s
index 77b444d520a1f10..b34e7142785d4a0 100644
--- a/bolt/test/X86/jump-table-func-entry.s
+++ b/bolt/test/X86/jump-table-func-entry.s
@@ -7,7 +7,7 @@
 # RUN: %clang %cflags %t.o -o %t.exe -no-pie -Wl,-q
 
 # RUN: llvm-bolt %t.exe --print-normalized --print-only=foo -o %t.out \
-# RUN:   |& FileCheck %s
+# RUN:   2>&1 | FileCheck %s
 
 
 
diff --git a/bolt/test/X86/keep-nops.s b/bolt/test/X86/keep-nops.s
index 37da2ff07b9b798..ddd34ce61d620de 100644
--- a/bolt/test/X86/keep-nops.s
+++ b/bolt/test/X86/keep-nops.s
@@ -5,7 +5,7 @@
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux %s -o %t.o
 # RUN: ld.lld %t.o -o %t.exe -q
 # RUN: llvm-bolt %t.exe -o %t.bolt.exe --keep-nops --relocs --print-finalized \
-# RUN:   |& FileCheck --check-prefix=CHECK-BOLT %s
+# RUN:   2>&1 | FileCheck --check-prefix=CHECK-BOLT %s
 # RUN: llvm-objdump -d %t.bolt.exe | FileCheck %s
 
   .text
diff --git a/bolt/test/X86/linux-bug-table.s b/bolt/test/X86/linux-bug-table.s
index 63f70a0b35d9fe5..07a4729ade73747 100644
--- a/bolt/test/X86/linux-bug-table.s
+++ b/bolt/test/X86/linux-bug-table.s
@@ -15,7 +15,7 @@
 ## Verify bug entry bindings again after unreachable code elimination.
 
 # RUN: llvm-bolt %t.out -o %t.out.1 --print-only=_start --print-normalized \
-# RUN:   |& FileCheck --check-prefix=CHECK-REOPT %s
+# RUN:   2>&1 | FileCheck --check-prefix=CHECK-REOPT %s
 
 # CHECK:      BOLT-INFO: Linux kernel binary detected
 # CHECK:      BOLT-INFO: parsed 2 bug table entries
diff --git a/bolt/test/X86/linux-orc.s b/bolt/test/X86/linux-orc.s
index 5f2096278e92d6a..1b0e681b1dbf96e 100644
--- a/bolt/test/X86/linux-orc.s
+++ b/bolt/test/X86/linux-orc.s
@@ -9,7 +9,7 @@
 
 ## Verify reading contents of ORC sections.
 
-# RUN: llvm-bolt %t.exe --dump-orc -o /dev/null |& FileCheck %s \
+# RUN: llvm-bolt %t.exe --dump-orc -o /dev/null 2>&1 | FileCheck %s \
 # RUN:   --check-prefix=CHECK-ORC
 
 # CHECK-ORC: 	    BOLT-INFO: ORC unwind information:
@@ -27,19 +27,19 @@
 ## Verify ORC bindings to instructions.
 
 # RUN: llvm-bolt %t.exe --print-normalized --dump-orc --print-orc -o %t.out \
-# RUN:   --keep-nops=0 --bolt-info=0 |& FileCheck %s
+# RUN:   --keep-nops=0 --bolt-info=0 2>&1 | FileCheck %s
 
 
 ## Verify ORC bindings after rewrite.
 
 # RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized --print-orc \
-# RUN:   |& FileCheck %s
+# RUN:   2>&1 | FileCheck %s
 
 ## Verify ORC binding after rewrite when some of the functions are skipped.
 
 # RUN: llvm-bolt %t.exe -o %t.out --skip-funcs=bar --bolt-info=0 --keep-nops=0
 # RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized --print-orc \
-# RUN:   |& FileCheck %s
+# RUN:   2>&1 | FileCheck %s
 
 # CHECK:      BOLT-INFO: Linux kernel binary detected
 # CHECK:      BOLT-INFO: parsed 9 ORC entries
diff --git a/bolt/test/X86/linux-pci-fixup.s b/bolt/test/X86/linux-pci-fixup.s
index a574ba84c4df11e..42504c108d339ce 100644
--- a/bolt/test/X86/linux-pci-fixup.s
+++ b/bolt/test/X86/linux-pci-fixup.s
@@ -3,7 +3,7 @@
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
 # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \
 # RUN:   -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.exe --print-normalized -o %t.out |& FileCheck %s
+# RUN: llvm-bolt %t.exe --print-normalized -o %t.out 2>&1 | FileCheck %s
 
 ## Check that BOLT correctly parses the Linux kernel .pci_fixup section and
 ## verify that PCI fixup hook in the middle of a function is detected.
diff --git a/bolt/test/X86/linux-smp-locks.s b/bolt/test/X86/linux-smp-locks.s
index 5f4410d14fc6b08..50d9e632b117205 100644
--- a/bolt/test/X86/linux-smp-locks.s
+++ b/bolt/test/X86/linux-smp-locks.s
@@ -7,11 +7,11 @@
 # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \
 # RUN:   -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
 # RUN: llvm-bolt %t.exe --print-normalized --keep-nops=0 --bolt-info=0 -o %t.out \
-# RUN:   |& FileCheck %s
+# RUN:   2>&1 | FileCheck %s
 
 ## Check the output of BOLT with NOPs removed.
 
-# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized |& FileCheck %s
+# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized 2>&1 | FileCheck %s
 
 # CHECK:      BOLT-INFO: Linux kernel binary detected
 # CHECK:      BOLT-INFO: parsed 2 SMP lock entries
diff --git a/bolt/test/X86/linux-static-calls.s b/bolt/test/X86/linux-static-calls.s
index caf95e1c03227d2..ce90f4bb79c094e 100644
--- a/bolt/test/X86/linux-static-calls.s
+++ b/bolt/test/X86/linux-static-calls.s
@@ -9,11 +9,11 @@
 ## Verify static calls bindings to instructions.
 
 # RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 \
-# RUN:   --bolt-info=0 |& FileCheck %s
+# RUN:   --bolt-info=0 2>&1 | FileCheck %s
 
 ## Verify the bindings again on the rewritten binary with nops removed.
 
-# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized |& FileCheck %s
+# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized 2>&1 | FileCheck %s
 
 # CHECK:      BOLT-INFO: Linux kernel binary detected
 # CHECK:      BOLT-INFO: parsed 2 static call entries
diff --git a/bolt/test/X86/linux-static-keys.s b/bolt/test/X86/linux-static-keys.s
index fb419e0f7627559..0bd17a375d88248 100644
--- a/bolt/test/X86/linux-static-keys.s
+++ b/bolt/test/X86/linux-static-keys.s
@@ -11,17 +11,17 @@
 ## Verify static keys jump bindings to instructions.
 
 # RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 \
-# RUN:   --bolt-info=0 |& FileCheck %s
+# RUN:   --bolt-info=0 2>&1 | FileCheck %s
 
 ## Verify that profile is matched correctly.
 
 # RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 \
-# RUN:   --bolt-info=0 --data %t.fdata |& \
-# RUN:   FileCheck --check-prefix=CHECK-FDATA %s
+# RUN:   --bolt-info=0 --data %t.fdata 2>&1 \
+# RUN:   | FileCheck --check-prefix=CHECK-FDATA %s
 
 ## Verify the bindings again on the rewritten binary with nops removed.
 
-# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized |& FileCheck %s
+# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized 2>&1 | FileCheck %s
 
 # CHECK:      BOLT-INFO: Linux kernel binary detected
 # CHECK:      BOLT-INFO: parsed 2 static keys jump entries
diff --git a/bolt/test/X86/pt_gnu_relro.s b/bolt/test/X86/pt_gnu_relro.s
index d7cfad5f954be5e..ff897b96e090970 100644
--- a/bolt/test/X86/pt_gnu_relro.s
+++ b/bolt/test/X86/pt_gnu_relro.s
@@ -22,7 +22,7 @@
 # READELF: 04 .got
 
 # RUN: llvm-bolt %t.exe --relocs -o %t.null -v=1 \
-# RUN:   |& FileCheck --check-prefix=BOLT %s
+# RUN:   2>&1 | FileCheck --check-prefix=BOLT %s
 # BOLT: BOLT-INFO: marking .got as GNU_RELRO
 
   .globl _start
diff --git a/bolt/test/X86/unclaimed-jt-entries.s b/bolt/test/X86/unclaimed-jt-entries.s
index 2d56167286c36bb..1102e4ae413e277 100644
--- a/bolt/test/X86/unclaimed-jt-entries.s
+++ b/bolt/test/X86/unclaimed-jt-entries.s
@@ -18,7 +18,7 @@
 
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
 # RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-q
-# RUN: llvm-bolt %t.exe -v=1 -o %t.out |& FileCheck %s
+# RUN: llvm-bolt %t.exe -v=1 -o %t.out 2>&1 | FileCheck %s
 
 # CHECK: BOLT-WARNING: unclaimed data to code reference (possibly an unrecognized jump table entry) to .Ltmp[[#]] in main
 # CHECK: BOLT-WARNING: unclaimed data to code reference (possibly an unrecognized jump table entry) to .Ltmp[[#]] in main
diff --git a/bolt/test/X86/vararg.test b/bolt/test/X86/vararg.test
index 0b8668a842ed4d5..296c739c6e11383 100644
--- a/bolt/test/X86/vararg.test
+++ b/bolt/test/X86/vararg.test
@@ -5,7 +5,7 @@
 REQUIRES: x86_64-linux
 
 RUN: %clangxx %cxxflags -no-pie %p/../Inputs/vararg.s -o %t -Wl,-q
-RUN: llvm-bolt %t -o %t.null --print-cfg --print-only=.*printf.* |& FileCheck %s
+RUN: llvm-bolt %t -o %t.null --print-cfg --print-only=.*printf.* 2>&1 | FileCheck %s
 
 CHECK: IsSimple    : 0
 CHECK: Entry Point
diff --git a/bolt/test/runtime/X86/unclaimed-jt-entries.s b/bolt/test/runtime/X86/unclaimed-jt-entries.s
index d0691d256ba0458..1725fb808efbff6 100644
--- a/bolt/test/runtime/X86/unclaimed-jt-entries.s
+++ b/bolt/test/runtime/X86/unclaimed-jt-entries.s
@@ -18,7 +18,7 @@
 
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
 # RUN: %clang %cflags %S/Inputs/unclaimed-jt-entries.c -no-pie %t.o -o %t.exe -Wl,-q
-# RUN: llvm-bolt %t.exe -v=1 -o %t.out --sequential-disassembly |& FileCheck %s
+# RUN: llvm-bolt %t.exe -v=1 -o %t.out --sequential-disassembly 2>&1 | FileCheck %s
 
 # CHECK: BOLT-WARNING: unclaimed data to code reference (possibly an unrecognized jump table entry) to .Ltmp[[#]] in func
 # CHECK: BOLT-WARNING: unclaimed data to code reference (possibly an unrecognized jump table entry) to .Ltmp[[#]] in func
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index aef22453035c30a..21ee417da6028a8 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -352,6 +352,7 @@ genHTML(const EnumInfo &I, const ClangDocContext &CDCtx);
 static std::vector<std::unique_ptr<TagNode>>
 genHTML(const FunctionInfo &I, const ClangDocContext &CDCtx,
         StringRef ParentInfoDir);
+static std::unique_ptr<TagNode> genHTML(const std::vector<CommentInfo> &C);
 
 static std::vector<std::unique_ptr<TagNode>>
 genEnumsBlock(const std::vector<EnumInfo> &Enums,
@@ -418,9 +419,13 @@ genRecordMembersBlock(const llvm::SmallVector<MemberTypeInfo, 4> &Members,
     if (Access != "")
       Access = Access + " ";
     auto LIBody = std::make_unique<TagNode>(HTMLTag::TAG_LI);
-    LIBody->Children.emplace_back(std::make_unique<TextNode>(Access));
-    LIBody->Children.emplace_back(genReference(M.Type, ParentInfoDir));
-    LIBody->Children.emplace_back(std::make_unique<TextNode>(" " + M.Name));
+    auto MemberDecl = std::make_unique<TagNode>(HTMLTag::TAG_DIV);
+    MemberDecl->Children.emplace_back(std::make_unique<TextNode>(Access));
+    MemberDecl->Children.emplace_back(genReference(M.Type, ParentInfoDir));
+    MemberDecl->Children.emplace_back(std::make_unique<TextNode>(" " + M.Name));
+    if (!M.Description.empty())
+      LIBody->Children.emplace_back(genHTML(M.Description));
+    LIBody->Children.emplace_back(std::move(MemberDecl));
     ULBody->Children.emplace_back(std::move(LIBody));
   }
   return Out;
diff --git a/clang-tools-extra/test/clang-doc/basic-project.test b/clang-tools-extra/test/clang-doc/basic-project.test
index c2c7548f5a9690c..2865ed4446e7ef8 100644
--- a/clang-tools-extra/test/clang-doc/basic-project.test
+++ b/clang-tools-extra/test/clang-doc/basic-project.test
@@ -91,8 +91,10 @@
 // HTML-RECTANGLE:   <a href="Shape.html">Shape</a>
 // HTML-RECTANGLE: </p>
 // HTML-RECTANGLE: <h2 id="Members">Members</h2>
-// HTML-RECTANGLE: <li>private double width_</li>
-// HTML-RECTANGLE: <li>private double height_</li>
+// HTML-RECTANGLE: <p> Width of the rectangle.</p>
+// HTML-RECTANGLE: <div>private double width_</div>
+// HTML-RECTANGLE: <p> Height of the rectangle.</p>
+// HTML-RECTANGLE: <div>private double height_</div>
 // HTML-RECTANGLE: <h2 id="Functions">Functions</h2>
 // HTML-RECTANGLE: <h3 id="{{([0-9A-F]{40})}}">Rectangle</h3>
 // HTML-RECTANGLE: <p>public void Rectangle(double width, double height)</p>
@@ -112,7 +114,8 @@
 // HTML-CIRCLE:   <a href="Shape.html">Shape</a>
 // HTML-CIRCLE: </p>
 // HTML-CIRCLE: <h2 id="Members">Members</h2>
-// HTML-CIRCLE: <li>private double radius_</li>
+// HTML-CIRCLE: <p> Radius of the circle.</p>
+// HTML-CIRCLE: <div>private double radius_</div>
 // HTML-CIRCLE: <h2 id="Functions">Functions</h2>
 // HTML-CIRCLE: <h3 id="{{([0-9A-F]{40})}}">Circle</h3>
 // HTML-CIRCLE: <p>public void Circle(double radius)</p>
diff --git a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
index e4a7340318b9341..bd031282b042abb 100644
--- a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
@@ -197,7 +197,9 @@ TEST(HTMLGeneratorTest, emitRecordHTML) {
     </p>
     <h2 id="Members">Members</h2>
     <ul>
-      <li>private int X</li>
+      <li>
+        <div>private int X</div>
+      </li>
     </ul>
     <h2 id="Records">Records</h2>
     <ul>
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6796a619ba97f8e..39e1b0fcb09bbd5 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -217,6 +217,8 @@ Bug Fixes to C++ Support
 - Clang now preserves the unexpanded flag in a lambda transform used for pack expansion. (#GH56852), (#GH85667),
   (#GH99877).
 - Fixed a bug when diagnosing ambiguous explicit specializations of constrained member functions.
+- Fixed an assertion failure when selecting a function from an overload set that includes a 
+  specialization of a conversion function template.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 94cf9113cd43a47..58a820508da42b6 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -34,6 +34,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -1194,8 +1195,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   llvm::DenseSet<const VarDecl *> CUDADeviceVarODRUsedByHost;
 
   /// Keep track of CUDA/HIP external kernels or device variables ODR-used by
-  /// host code.
-  llvm::DenseSet<const ValueDecl *> CUDAExternalDeviceDeclODRUsedByHost;
+  /// host code. SetVector is used to maintain the order.
+  llvm::SetVector<const ValueDecl *> CUDAExternalDeviceDeclODRUsedByHost;
 
   /// Keep track of CUDA/HIP implicit host device functions used on device side
   /// in device compilation.
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index a024f9b2a9f8c0d..eb82e0159b56ede 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -166,15 +166,22 @@ def note_constexpr_heap_alloc_limit_exceeded : Note<
 def note_constexpr_this : Note<
   "%select{|implicit }0use of 'this' pointer is only allowed within the "
   "evaluation of a call to a 'constexpr' member function">;
-def note_constexpr_lifetime_ended : Note<
+def access_kind : TextSubstitution<
   "%select{read of|read of|assignment to|increment of|decrement of|"
   "member call on|dynamic_cast of|typeid applied to|construction of|"
-  "destruction of}0 %select{temporary|variable}1 whose "
-  "%plural{8:storage duration|:lifetime}0 has ended">;
-def note_constexpr_access_uninit : Note<
+  "destruction of}0">;
+def access_kind_subobject : TextSubstitution<
   "%select{read of|read of|assignment to|increment of|decrement of|"
   "member call on|dynamic_cast of|typeid applied to|"
-  "construction of subobject of|destruction of}0 "
+  "construction of subobject of|destruction of}0">;
+def access_kind_volatile : TextSubstitution<
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "<ERROR>|<ERROR>|<ERROR>|<ERROR>|<ERROR>}0">;
+def note_constexpr_lifetime_ended : Note<
+  "%sub{access_kind}0 %select{temporary|variable}1 whose "
+  "%plural{8:storage duration|:lifetime}0 has ended">;
+def note_constexpr_access_uninit : Note<
+  "%sub{access_kind_subobject}0 "
   "%select{object outside its lifetime|uninitialized object}1 "
   "is not allowed in a constant expression">;
 def note_constexpr_use_uninit_reference : Note<
@@ -184,20 +191,16 @@ def note_constexpr_modify_const_type : Note<
   "modification of object of const-qualified type %0 is not allowed "
   "in a constant expression">;
 def note_constexpr_access_volatile_type : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "<ERROR>|<ERROR>|<ERROR>|<ERROR>}0 "
+  "%sub{access_kind_volatile}0 "
   "volatile-qualified type %1 is not allowed in a constant expression">;
 def note_constexpr_access_volatile_obj : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "<ERROR>|<ERROR>|<ERROR>|<ERROR>}0 "
+  "%sub{access_kind_volatile}0 "
   "volatile %select{temporary|object %2|member %2}1 is not allowed in "
   "a constant expression">;
 def note_constexpr_volatile_here : Note<
   "volatile %select{temporary created|object declared|member declared}0 here">;
 def note_constexpr_access_mutable : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|construction of|"
-  "destruction of}0 "
+  "%sub{access_kind}0 "
   "mutable member %1 is not allowed in a constant expression">;
 def note_constexpr_ltor_non_const_int : Note<
   "read of non-const variable %0 is not allowed in a constant expression">;
@@ -209,47 +212,28 @@ def note_constexpr_ltor_non_constexpr : Note<
 def note_constexpr_ltor_incomplete_type : Note<
   "read of incomplete type %0 is not allowed in a constant expression">;
 def note_constexpr_access_null : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|construction of|"
-  "destruction of}0 "
+  "%sub{access_kind}0 "
   "dereferenced null pointer is not allowed in a constant expression">;
 def note_constexpr_access_past_end : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|construction of|"
-  "destruction of}0 "
-  "dereferenced one-past-the-end pointer is not allowed "
-  "in a constant expression">;
+  "%sub{access_kind}0 dereferenced one-past-the-end pointer "
+  "is not allowed in a constant expression">;
 def note_constexpr_access_unsized_array : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|construction of|"
-  "destruction of}0 "
-  "element of array without known bound "
+  "%sub{access_kind}0 element of array without known bound "
   "is not allowed in a constant expression">;
 def note_constexpr_access_inactive_union_member : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|"
-  "construction of subobject of|destruction of}0 "
+  "%sub{access_kind_subobject}0 "
   "member %1 of union with %select{active member %3|no active member}2 "
   "is not allowed in a constant expression">;
 def note_constexpr_union_member_change_during_init : Note<
   "assignment would change active union member during the initialization of "
   "a different member of the same union">;
 def note_constexpr_access_static_temporary : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|reconstruction of|"
-  "destruction of}0 temporary "
-  "is not allowed in a constant expression outside the expression that "
-  "created the temporary">;
+  "%sub{access_kind}0 temporary is not allowed in a constant expression "
+  "outside the expression that created the temporary">;
 def note_constexpr_access_unreadable_object : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|construction of|"
-  "destruction of}0 "
-  "object '%1' whose value is not known">;
+  "%sub{access_kind}0 object '%1' whose value is not known">;
 def note_constexpr_access_deleted_object : Note<
-  "%select{read of|read of|assignment to|increment of|decrement of|"
-  "member call on|dynamic_cast of|typeid applied to|construction of|"
-  "destruction of}0 "
-  "heap allocated object that has been deleted">;
+  "%sub{access_kind}0 heap allocated object that has been deleted">;
 def note_constexpr_modify_global : Note<
   "a constant expression cannot modify an object that is visible outside "
   "that expression">;
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index f8d50d12bb9351d..12aab09f285567f 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1260,9 +1260,6 @@ def warn_pragma_intrinsic_builtin : Warning<
 def warn_pragma_unused_expected_var : Warning<
   "expected '#pragma unused' argument to be a variable name">,
   InGroup<IgnoredPragmas>;
-// - #pragma mc_func
-def err_pragma_mc_func_not_supported :
-   Error<"#pragma mc_func is not supported">;
 // - #pragma init_seg
 def warn_pragma_init_seg_unsupported_target : Warning<
   "'#pragma init_seg' is only supported when targeting a "
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index e196c3dc5cb3be8..0b38139bd279725 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -8114,13 +8114,6 @@ def source_date_epoch : Separate<["-"], "source-date-epoch">,
 
 } // let Visibility = [CC1Option]
 
-defm err_pragma_mc_func_aix : BoolFOption<"err-pragma-mc-func-aix",
-  PreprocessorOpts<"ErrorOnPragmaMcfuncOnAIX">, DefaultFalse,
-  PosFlag<SetTrue, [], [ClangOption, CC1Option],
-          "Treat uses of #pragma mc_func as errors">,
-  NegFlag<SetFalse,[], [ClangOption, CC1Option],
-          "Ignore uses of #pragma mc_func">>;
-
 //===----------------------------------------------------------------------===//
 // CUDA Options
 //===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/Lex/PreprocessorOptions.h b/clang/include/clang/Lex/PreprocessorOptions.h
index 3f7dd9db18ba7d1..c2e3d68333024a5 100644
--- a/clang/include/clang/Lex/PreprocessorOptions.h
+++ b/clang/include/clang/Lex/PreprocessorOptions.h
@@ -211,10 +211,6 @@ class PreprocessorOptions {
   /// If set, the UNIX timestamp specified by SOURCE_DATE_EPOCH.
   std::optional<uint64_t> SourceDateEpoch;
 
-  /// If set, the preprocessor reports an error when processing #pragma mc_func
-  /// on AIX.
-  bool ErrorOnPragmaMcfuncOnAIX = false;
-
 public:
   PreprocessorOptions() : PrecompiledPreambleBytes(0, false) {}
 
@@ -252,7 +248,6 @@ class PreprocessorOptions {
     PrecompiledPreambleBytes.first = 0;
     PrecompiledPreambleBytes.second = false;
     RetainExcludedConditionalBlocks = false;
-    ErrorOnPragmaMcfuncOnAIX = false;
   }
 };
 
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 39c5f588167edeb..99a0b0200fa06fe 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -221,7 +221,6 @@ class Parser : public CodeCompletionHandler {
   std::unique_ptr<PragmaHandler> MaxTokensHerePragmaHandler;
   std::unique_ptr<PragmaHandler> MaxTokensTotalPragmaHandler;
   std::unique_ptr<PragmaHandler> RISCVPragmaHandler;
-  std::unique_ptr<PragmaHandler> MCFuncPragmaHandler;
 
   std::unique_ptr<CommentHandler> CommentSemaHandler;
 
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index 13390007fde33c8..4a50b4487b66544 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -126,13 +126,17 @@ static bool CheckActive(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
     return true;
 
   assert(Ptr.inUnion());
+  assert(Ptr.isField() && Ptr.getField());
 
   Pointer U = Ptr.getBase();
   Pointer C = Ptr;
   while (!U.isRoot() && U.inUnion() && !U.isActive()) {
-    C = U;
+    if (U.getField())
+      C = U;
     U = U.getBase();
   }
+  assert(C.isField());
+
   // Get the inactive field descriptor.
   const FieldDecl *InactiveField = C.getField();
   assert(InactiveField);
diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/Interp/InterpBuiltin.cpp
index 1841a2a4714d89d..c3370e2e5286e07 100644
--- a/clang/lib/AST/Interp/InterpBuiltin.cpp
+++ b/clang/lib/AST/Interp/InterpBuiltin.cpp
@@ -1635,7 +1635,58 @@ bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
   return true;
 }
 
-bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
+static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
+                          Pointer &Dest, bool Activate);
+static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
+                       Pointer &Dest, bool Activate = false) {
+  [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
+  const Descriptor *DestDesc = Dest.getFieldDesc();
+
+  auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
+    Pointer DestField = Dest.atField(F.Offset);
+    if (std::optional<PrimType> FT = S.Ctx.classify(F.Decl->getType())) {
+      TYPE_SWITCH(*FT, {
+        DestField.deref<T>() = Src.atField(F.Offset).deref<T>();
+        if (Src.atField(F.Offset).isInitialized())
+          DestField.initialize();
+        if (Activate)
+          DestField.activate();
+      });
+      return true;
+    }
+    // Composite field.
+    return copyComposite(S, OpPC, Src.atField(F.Offset), DestField, Activate);
+  };
+
+  assert(SrcDesc->isRecord());
+  assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
+  const Record *R = DestDesc->ElemRecord;
+  for (const Record::Field &F : R->fields()) {
+    if (R->isUnion()) {
+      // For unions, only copy the active field.
+      const Pointer &SrcField = Src.atField(F.Offset);
+      if (SrcField.isActive()) {
+        if (!copyField(F, /*Activate=*/true))
+          return false;
+      }
+    } else {
+      if (!copyField(F, Activate))
+        return false;
+    }
+  }
+
+  for (const Record::Base &B : R->bases()) {
+    Pointer DestBase = Dest.atField(B.Offset);
+    if (!copyRecord(S, OpPC, Src.atField(B.Offset), DestBase, Activate))
+      return false;
+  }
+
+  Dest.initialize();
+  return true;
+}
+
+static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
+                          Pointer &Dest, bool Activate = false) {
   assert(Src.isLive() && Dest.isLive());
 
   [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
@@ -1657,44 +1708,14 @@ bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
     return true;
   }
 
-  if (DestDesc->isRecord()) {
-    auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
-      Pointer DestField = Dest.atField(F.Offset);
-      if (std::optional<PrimType> FT = S.Ctx.classify(F.Decl->getType())) {
-        TYPE_SWITCH(*FT, {
-          DestField.deref<T>() = Src.atField(F.Offset).deref<T>();
-          DestField.initialize();
-          if (Activate)
-            DestField.activate();
-        });
-        return true;
-      }
-      return Invalid(S, OpPC);
-    };
-
-    assert(SrcDesc->isRecord());
-    assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
-    const Record *R = DestDesc->ElemRecord;
-    for (const Record::Field &F : R->fields()) {
-      if (R->isUnion()) {
-        // For unions, only copy the active field.
-        const Pointer &SrcField = Src.atField(F.Offset);
-        if (SrcField.isActive()) {
-          if (!copyField(F, /*Activate=*/true))
-            return false;
-        }
-      } else {
-        if (!copyField(F, /*Activate=*/false))
-          return false;
-      }
-    }
-    return true;
-  }
-
-  // FIXME: Composite types.
-
+  if (DestDesc->isRecord())
+    return copyRecord(S, OpPC, Src, Dest, Activate);
   return Invalid(S, OpPC);
 }
 
+bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
+  return copyComposite(S, OpPC, Src, Dest);
+}
+
 } // namespace interp
 } // namespace clang
diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp
index b2885b7776d1321..c2de7328c25c5d9 100644
--- a/clang/lib/Driver/ToolChains/AIX.cpp
+++ b/clang/lib/Driver/ToolChains/AIX.cpp
@@ -560,12 +560,6 @@ void AIX::addClangTargetOptions(
   if (!Args.getLastArgNoClaim(options::OPT_fsized_deallocation,
                               options::OPT_fno_sized_deallocation))
     CC1Args.push_back("-fno-sized-deallocation");
-
-  if (Args.hasFlag(options::OPT_ferr_pragma_mc_func_aix,
-                   options::OPT_fno_err_pragma_mc_func_aix, false))
-    CC1Args.push_back("-ferr-pragma-mc-func-aix");
-  else
-    CC1Args.push_back("-fno-err-pragma-mc-func-aix");
 }
 
 void AIX::addProfileRTLibs(const llvm::opt::ArgList &Args,
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index aef4ddb75881644..cc6f18b5b319f95 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -14,7 +14,6 @@
 #include "clang/Basic/PragmaKinds.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/Preprocessor.h"
-#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Lex/Token.h"
 #include "clang/Parse/LoopHint.h"
 #include "clang/Parse/ParseDiagnostic.h"
@@ -412,19 +411,6 @@ struct PragmaRISCVHandler : public PragmaHandler {
   Sema &Actions;
 };
 
-struct PragmaMCFuncHandler : public PragmaHandler {
-  PragmaMCFuncHandler(bool ReportError)
-      : PragmaHandler("mc_func"), ReportError(ReportError) {}
-  void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
-                    Token &Tok) override {
-    if (ReportError)
-      PP.Diag(Tok, diag::err_pragma_mc_func_not_supported);
-  }
-
-private:
-  bool ReportError = false;
-};
-
 void markAsReinjectedForRelexing(llvm::MutableArrayRef<clang::Token> Toks) {
   for (auto &T : Toks)
     T.setFlag(clang::Token::IsReinjected);
@@ -582,12 +568,6 @@ void Parser::initializePragmaHandlers() {
     RISCVPragmaHandler = std::make_unique<PragmaRISCVHandler>(Actions);
     PP.AddPragmaHandler("clang", RISCVPragmaHandler.get());
   }
-
-  if (getTargetInfo().getTriple().isOSAIX()) {
-    MCFuncPragmaHandler = std::make_unique<PragmaMCFuncHandler>(
-        PP.getPreprocessorOpts().ErrorOnPragmaMcfuncOnAIX);
-    PP.AddPragmaHandler(MCFuncPragmaHandler.get());
-  }
 }
 
 void Parser::resetPragmaHandlers() {
@@ -722,11 +702,6 @@ void Parser::resetPragmaHandlers() {
     PP.RemovePragmaHandler("clang", RISCVPragmaHandler.get());
     RISCVPragmaHandler.reset();
   }
-
-  if (getTargetInfo().getTriple().isOSAIX()) {
-    PP.RemovePragmaHandler(MCFuncPragmaHandler.get());
-    MCFuncPragmaHandler.reset();
-  }
 }
 
 /// Handle the annotation token produced for #pragma unused(...)
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index e9705ec43d86cc6..ec951d5ac06dbc6 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -5805,12 +5805,19 @@ FunctionDecl *Sema::getMoreConstrainedFunction(FunctionDecl *FD1,
                                                FunctionDecl *FD2) {
   assert(!FD1->getDescribedTemplate() && !FD2->getDescribedTemplate() &&
          "not for function templates");
+  assert(!FD1->isFunctionTemplateSpecialization() ||
+         isa<CXXConversionDecl>(FD1));
+  assert(!FD2->isFunctionTemplateSpecialization() ||
+         isa<CXXConversionDecl>(FD2));
+
   FunctionDecl *F1 = FD1;
-  if (FunctionDecl *MF = FD1->getInstantiatedFromMemberFunction())
-    F1 = MF;
+  if (FunctionDecl *P = FD1->getTemplateInstantiationPattern(false))
+    F1 = P;
+
   FunctionDecl *F2 = FD2;
-  if (FunctionDecl *MF = FD2->getInstantiatedFromMemberFunction())
-    F2 = MF;
+  if (FunctionDecl *P = FD2->getTemplateInstantiationPattern(false))
+    F2 = P;
+
   llvm::SmallVector<const Expr *, 1> AC1, AC2;
   F1->getAssociatedConstraints(AC1);
   F2->getAssociatedConstraints(AC2);
diff --git a/clang/test/AST/Interp/unions.cpp b/clang/test/AST/Interp/unions.cpp
index 996d29e143fe2c8..35b4a520baa2695 100644
--- a/clang/test/AST/Interp/unions.cpp
+++ b/clang/test/AST/Interp/unions.cpp
@@ -361,7 +361,7 @@ namespace CopyCtor {
 
 namespace UnionInBase {
   struct Base {
-    int y;
+    int y; // both-note {{subobject declared here}}
   };
   struct A : Base {
     int x;
@@ -380,5 +380,29 @@ namespace UnionInBase {
   }
   static_assert(read_wrong_member_indirect() == 1); // both-error {{not an integral constant expression}} \
                                                     // both-note {{in call to}}
+  constexpr int read_uninitialized() {
+    B b = {.b = 1};
+    int *p = &b.a.y;
+    b.a.x = 1;
+    return *p; // both-note {{read of uninitialized object}}
+  }
+  static_assert(read_uninitialized() == 0); // both-error {{constant}} \
+                                            // both-note {{in call}}
+  constexpr int write_uninitialized() {
+    B b = {.b = 1};
+    int *p = &b.a.y;
+    b.a.x = 1;
+    *p = 1;
+    return *p;
+  }
+
+  constexpr B return_uninit() {
+    B b = {.b = 1};
+    b.a.x = 2;
+    return b;
+  }
+  constexpr B uninit = return_uninit(); // both-error {{constant expression}} \
+                                        // both-note {{subobject 'y' is not initialized}}
+  static_assert(return_uninit().a.x == 2);
 }
 #endif
diff --git a/clang/test/CodeGenCUDA/host-used-extern-determinism.cu b/clang/test/CodeGenCUDA/host-used-extern-determinism.cu
new file mode 100644
index 000000000000000..1e52887b894b17a
--- /dev/null
+++ b/clang/test/CodeGenCUDA/host-used-extern-determinism.cu
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fcuda-is-device -x hip %s \
+// RUN:   -fgpu-rdc -std=c++11 -emit-llvm -o - -target-cpu gfx906 | FileCheck %s
+
+#include "Inputs/cuda.h"
+
+// CHECK-LABEL: @__clang_gpu_used_external = internal {{.*}}global
+// References to the kernels must be in order of appearance.
+// CHECK-SAME: [ptr @_Z6kernelILi3EEvPi, ptr @_Z6kernelILi1EEvPi, ptr @_Z6kernelILi2EEvPi, ptr @_Z6kernelILi0EEvPi]
+
+template <int N>
+__global__ void kernel(int* out) { *out = N; }
+
+void host(int n) {
+    void * k;
+    switch (n) {
+        case 3: k = (void*)&kernel<3>; break;
+        case 1: k = (void*)&kernel<1>; break;
+        case 2: k = (void*)&kernel<2>; break;
+        case 0: k = (void*)&kernel<0>; break;
+    }
+}
diff --git a/clang/test/Preprocessor/pragma_mc_func.c b/clang/test/Preprocessor/pragma_mc_func.c
deleted file mode 100644
index f0d3e49e5dddcaf..000000000000000
--- a/clang/test/Preprocessor/pragma_mc_func.c
+++ /dev/null
@@ -1,23 +0,0 @@
-// RUN: not %clang --target=powerpc64-ibm-aix -ferr-pragma-mc-func-aix -fsyntax-only \
-// RUN:   %s 2>&1 | FileCheck %s
-#pragma mc_func asm_barrier {"60000000"}
-
-// CHECK:  error: #pragma mc_func is not supported
-
-// Cases where no errors occur.
-// RUN: %clang --target=powerpc64-ibm-aix -fno-err-pragma-mc-func-aix -fsyntax-only %s
-// RUN: %clang --target=powerpc64-ibm-aix -ferr-pragma-mc-func-aix -fsyntax-only \
-// RUN:    -fno-err-pragma-mc-func-aix %s
-// RUN: %clang --target=powerpc64-ibm-aix -fsyntax-only %s
-// RUN: %clang --target=powerpc64-ibm-aix -Werror=unknown-pragmas \
-// RUN:   -fno-err-pragma-mc-func-aix -fsyntax-only %s
-
-// Cases where we have errors or warnings.
-// RUN: not %clang --target=powerpc64le-unknown-linux-gnu \
-// RUN:   -Werror=unknown-pragmas -fno-err-pragma-mc-func-aix -fsyntax-only %s 2>&1 | \
-// RUN:   FileCheck --check-prefix=UNUSED %s
-// RUN: %clang --target=powerpc64le-unknown-linux-gnu \
-// RUN:   -fno-err-pragma-mc-func-aix -fsyntax-only %s 2>&1 | \
-// RUN:   FileCheck --check-prefix=UNUSED %s
-
-// UNUSED: clang: warning: argument unused during compilation: '-fno-err-pragma-mc-func-aix' [-Wunused-command-line-argument]
diff --git a/clang/test/SemaCXX/PR98671.cpp b/clang/test/SemaCXX/PR98671.cpp
new file mode 100644
index 000000000000000..f50518673588520
--- /dev/null
+++ b/clang/test/SemaCXX/PR98671.cpp
@@ -0,0 +1,28 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only %s -verify
+
+struct S1 {
+  operator int();
+
+  template <typename T>
+  operator T();
+};
+
+
+// Ensure that no assertion is raised when overload resolution fails while
+// choosing between an operator function template and an operator function.
+constexpr auto r = &S1::operator int;
+// expected-error@-1 {{initializer of type '<overloaded function type>'}}
+
+
+template <typename T>
+struct S2 {
+  template <typename U=T>
+    S2(U={}) requires (sizeof(T) > 0) {}
+    // expected-note@-1 {{candidate constructor}}
+
+  template <typename U=T>
+    S2(U={}) requires (true) {}
+    // expected-note@-1 {{candidate constructor}}
+};
+
+S2<int> s;  // expected-error {{call to constructor of 'S2<int>' is ambiguous}}
diff --git a/compiler-rt/lib/nsan/CMakeLists.txt b/compiler-rt/lib/nsan/CMakeLists.txt
index fa9f02abdf0801c..2846f0292307b3a 100644
--- a/compiler-rt/lib/nsan/CMakeLists.txt
+++ b/compiler-rt/lib/nsan/CMakeLists.txt
@@ -4,9 +4,11 @@ include_directories(..)
 
 set(NSAN_SOURCES
   nsan.cpp
+  nsan_allocator.cpp
   nsan_flags.cpp
   nsan_interceptors.cpp
   nsan_malloc_linux.cpp
+  nsan_new_delete.cpp
   nsan_stats.cpp
   nsan_suppressions.cpp
   nsan_thread.cpp
diff --git a/compiler-rt/lib/nsan/nsan.cpp b/compiler-rt/lib/nsan/nsan.cpp
index 7d10681a1bc9178..bfa55c317cfe79e 100644
--- a/compiler-rt/lib/nsan/nsan.cpp
+++ b/compiler-rt/lib/nsan/nsan.cpp
@@ -807,6 +807,7 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
   if (nsan_initialized)
     return;
   nsan_init_is_running = true;
+  SanitizerToolName = "NumericalStabilitySanitizer";
 
   InitializeFlags();
   InitializeSuppressions();
@@ -814,11 +815,12 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __nsan_init() {
 
   DisableCoreDumperIfNecessary();
 
-  if (!MmapFixedNoReserve(TypesAddr(), UnusedAddr() - TypesAddr()))
+  if (!MmapFixedNoReserve(TypesAddr(), AllocatorAddr() - TypesAddr()))
     Die();
 
   InitializeInterceptors();
   NsanTSDInit(NsanTSDDtor);
+  NsanAllocatorInit();
 
   NsanThread *main_thread = NsanThread::Create(nullptr, nullptr);
   SetCurrentThread(main_thread);
diff --git a/compiler-rt/lib/nsan/nsan.h b/compiler-rt/lib/nsan/nsan.h
index 4e88ef4c00974d5..08dd02746be65a0 100644
--- a/compiler-rt/lib/nsan/nsan.h
+++ b/compiler-rt/lib/nsan/nsan.h
@@ -51,6 +51,14 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
 __nsan_default_options();
 }
 
+// Unwind the stack for fatal error, as the parameter `stack` is
+// empty without origins.
+#define GET_FATAL_STACK_TRACE_IF_EMPTY(STACK)                                  \
+  if (nsan_initialized && (STACK)->size == 0) {                                \
+    (STACK)->Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr,  \
+                    common_flags()->fast_unwind_on_fatal);                     \
+  }
+
 namespace __nsan {
 
 extern bool nsan_initialized;
diff --git a/compiler-rt/lib/nsan/nsan_allocator.cpp b/compiler-rt/lib/nsan/nsan_allocator.cpp
new file mode 100644
index 000000000000000..19004ad7dc8dbe7
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_allocator.cpp
@@ -0,0 +1,340 @@
+//===- nsan_allocator.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// NumericalStabilitySanitizer allocator.
+//
+//===----------------------------------------------------------------------===//
+
+#include "nsan_allocator.h"
+#include "interception/interception.h"
+#include "nsan.h"
+#include "nsan_flags.h"
+#include "nsan_platform.h"
+#include "nsan_thread.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+
+using namespace __nsan;
+
+DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
+DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
+
+namespace {
+struct Metadata {
+  uptr requested_size;
+};
+
+struct NsanMapUnmapCallback {
+  void OnMap(uptr p, uptr size) const {}
+  void OnMapSecondary(uptr p, uptr size, uptr user_begin,
+                      uptr user_size) const {}
+  void OnUnmap(uptr p, uptr size) const {}
+};
+
+const uptr kMaxAllowedMallocSize = 1ULL << 40;
+
+// Allocator64 parameters. Deliberately using a short name.
+struct AP64 {
+  static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
+  static const uptr kSpaceSize = 0x40000000000; // 4T.
+  static const uptr kMetadataSize = sizeof(Metadata);
+  using SizeClassMap = DefaultSizeClassMap;
+  using MapUnmapCallback = NsanMapUnmapCallback;
+  static const uptr kFlags = 0;
+  using AddressSpaceView = LocalAddressSpaceView;
+};
+} // namespace
+
+using PrimaryAllocator = SizeClassAllocator64<AP64>;
+using Allocator = CombinedAllocator<PrimaryAllocator>;
+using AllocatorCache = Allocator::AllocatorCache;
+
+static Allocator allocator;
+static AllocatorCache fallback_allocator_cache;
+static StaticSpinMutex fallback_mutex;
+
+static uptr max_malloc_size;
+
+void __nsan::NsanAllocatorInit() {
+  SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+  allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+  if (common_flags()->max_allocation_size_mb)
+    max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
+                          kMaxAllowedMallocSize);
+  else
+    max_malloc_size = kMaxAllowedMallocSize;
+}
+
+static AllocatorCache *GetAllocatorCache(NsanThreadLocalMallocStorage *ms) {
+  CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
+  return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
+}
+
+void NsanThreadLocalMallocStorage::Init() {
+  allocator.InitCache(GetAllocatorCache(this));
+}
+
+void NsanThreadLocalMallocStorage::CommitBack() {
+  allocator.SwallowCache(GetAllocatorCache(this));
+  allocator.DestroyCache(GetAllocatorCache(this));
+}
+
+static void *NsanAllocate(uptr size, uptr alignment, bool zero) {
+  if (UNLIKELY(size > max_malloc_size)) {
+    if (AllocatorMayReturnNull()) {
+      Report("WARNING: NumericalStabilitySanitizer failed to allocate 0x%zx "
+             "bytes\n",
+             size);
+      return nullptr;
+    }
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
+  }
+  if (UNLIKELY(IsRssLimitExceeded())) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportRssLimitExceeded(&stack);
+  }
+
+  void *allocated;
+  if (NsanThread *t = GetCurrentThread()) {
+    AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+    allocated = allocator.Allocate(cache, size, alignment);
+  } else {
+    SpinMutexLock l(&fallback_mutex);
+    AllocatorCache *cache = &fallback_allocator_cache;
+    allocated = allocator.Allocate(cache, size, alignment);
+  }
+  if (UNLIKELY(!allocated)) {
+    SetAllocatorOutOfMemory();
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportOutOfMemory(size, &stack);
+  }
+  auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+  meta->requested_size = size;
+  if (zero && allocator.FromPrimary(allocated))
+    REAL(memset)(allocated, 0, size);
+  __nsan_set_value_unknown(allocated, size);
+  RunMallocHooks(allocated, size);
+  return allocated;
+}
+
+void __nsan::NsanDeallocate(void *p) {
+  DCHECK(p);
+  RunFreeHooks(p);
+  auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
+  uptr size = meta->requested_size;
+  meta->requested_size = 0;
+  if (flags().poison_in_free)
+    __nsan_set_value_unknown(p, size);
+  if (NsanThread *t = GetCurrentThread()) {
+    AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+    allocator.Deallocate(cache, p);
+  } else {
+    // In a just created thread, glibc's _dl_deallocate_tls might reach here
+    // before nsan_current_thread is set.
+    SpinMutexLock l(&fallback_mutex);
+    AllocatorCache *cache = &fallback_allocator_cache;
+    allocator.Deallocate(cache, p);
+  }
+}
+
+static void *NsanReallocate(void *ptr, uptr new_size, uptr alignment) {
+  Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(ptr));
+  uptr old_size = meta->requested_size;
+  uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(ptr);
+  if (new_size <= actually_allocated_size) {
+    // We are not reallocating here.
+    meta->requested_size = new_size;
+    if (new_size > old_size)
+      __nsan_set_value_unknown((u8 *)ptr + old_size, new_size - old_size);
+    return ptr;
+  }
+  void *new_p = NsanAllocate(new_size, alignment, false);
+  if (new_p) {
+    uptr memcpy_size = Min(new_size, old_size);
+    REAL(memcpy)(new_p, ptr, memcpy_size);
+    __nsan_copy_values(new_p, ptr, memcpy_size);
+    NsanDeallocate(ptr);
+  }
+  return new_p;
+}
+
+static void *NsanCalloc(uptr nmemb, uptr size) {
+  if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportCallocOverflow(nmemb, size, &stack);
+  }
+  return NsanAllocate(nmemb * size, sizeof(u64), true);
+}
+
+static const void *AllocationBegin(const void *p) {
+  if (!p)
+    return nullptr;
+  void *beg = allocator.GetBlockBegin(p);
+  if (!beg)
+    return nullptr;
+  auto *b = reinterpret_cast<Metadata *>(allocator.GetMetaData(beg));
+  if (!b)
+    return nullptr;
+  if (b->requested_size == 0)
+    return nullptr;
+
+  return beg;
+}
+
+static uptr AllocationSizeFast(const void *p) {
+  return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;
+}
+
+static uptr AllocationSize(const void *p) {
+  if (!p)
+    return 0;
+  if (allocator.GetBlockBegin(p) != p)
+    return 0;
+  return AllocationSizeFast(p);
+}
+
+void *__nsan::nsan_malloc(uptr size) {
+  return SetErrnoOnNull(NsanAllocate(size, sizeof(u64), false));
+}
+
+void *__nsan::nsan_calloc(uptr nmemb, uptr size) {
+  return SetErrnoOnNull(NsanCalloc(nmemb, size));
+}
+
+void *__nsan::nsan_realloc(void *ptr, uptr size) {
+  if (!ptr)
+    return SetErrnoOnNull(NsanAllocate(size, sizeof(u64), false));
+  if (size == 0) {
+    NsanDeallocate(ptr);
+    return nullptr;
+  }
+  return SetErrnoOnNull(NsanReallocate(ptr, size, sizeof(u64)));
+}
+
+void *__nsan::nsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
+  if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+    errno = errno_ENOMEM;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportReallocArrayOverflow(nmemb, size, &stack);
+  }
+  return nsan_realloc(ptr, nmemb * size);
+}
+
+void *__nsan::nsan_valloc(uptr size) {
+  return SetErrnoOnNull(NsanAllocate(size, GetPageSizeCached(), false));
+}
+
+void *__nsan::nsan_pvalloc(uptr size) {
+  uptr PageSize = GetPageSizeCached();
+  if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+    errno = errno_ENOMEM;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportPvallocOverflow(size, &stack);
+  }
+  // pvalloc(0) should allocate one page.
+  size = size ? RoundUpTo(size, PageSize) : PageSize;
+  return SetErrnoOnNull(NsanAllocate(size, PageSize, false));
+}
+
+void *__nsan::nsan_aligned_alloc(uptr alignment, uptr size) {
+  if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+    errno = errno_EINVAL;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
+  }
+  return SetErrnoOnNull(NsanAllocate(size, alignment, false));
+}
+
+void *__nsan::nsan_memalign(uptr alignment, uptr size) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    if (AllocatorMayReturnNull())
+      return nullptr;
+    BufferedStackTrace stack;
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);
+    ReportInvalidAllocationAlignment(alignment, &stack);
+  }
+  return SetErrnoOnNull(NsanAllocate(size, alignment, false));
+}
+
+int __nsan::nsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
+  if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+    if (AllocatorMayReturnNull())
+      return errno_EINVAL;
+    BufferedStackTrace stack;
+    ReportInvalidPosixMemalignAlignment(alignment, &stack);
+  }
+  void *ptr = NsanAllocate(size, alignment, false);
+  if (UNLIKELY(!ptr))
+    // OOM error is already taken care of by NsanAllocate.
+    return errno_ENOMEM;
+  DCHECK(IsAligned((uptr)ptr, alignment));
+  *memptr = ptr;
+  return 0;
+}
+
+extern "C" {
+uptr __sanitizer_get_current_allocated_bytes() {
+  uptr stats[AllocatorStatCount];
+  allocator.GetStats(stats);
+  return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+  uptr stats[AllocatorStatCount];
+  allocator.GetStats(stats);
+  return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() { return 1; }
+
+uptr __sanitizer_get_unmapped_bytes() { return 1; }
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
+
+const void *__sanitizer_get_allocated_begin(const void *p) {
+  return AllocationBegin(p);
+}
+
+uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
+
+uptr __sanitizer_get_allocated_size_fast(const void *p) {
+  DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
+  uptr ret = AllocationSizeFast(p);
+  DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
+  return ret;
+}
+
+void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
+}
diff --git a/compiler-rt/lib/nsan/nsan_allocator.h b/compiler-rt/lib/nsan/nsan_allocator.h
new file mode 100644
index 000000000000000..d41560493c1a74c
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_allocator.h
@@ -0,0 +1,41 @@
+//===-- nsan_allocator.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef NSAN_ALLOCATOR_H
+#define NSAN_ALLOCATOR_H
+
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __nsan {
+
+struct NsanThreadLocalMallocStorage {
+  // Allocator cache contains atomic_uint64_t which must be 8-byte aligned.
+  alignas(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque.
+  void Init();
+  void CommitBack();
+
+private:
+  // These objects are allocated via mmap() and are zero-initialized.
+  NsanThreadLocalMallocStorage() {}
+};
+
+void NsanAllocatorInit();
+void NsanDeallocate(void *ptr);
+
+void *nsan_malloc(uptr size);
+void *nsan_calloc(uptr nmemb, uptr size);
+void *nsan_realloc(void *ptr, uptr size);
+void *nsan_reallocarray(void *ptr, uptr nmemb, uptr size);
+void *nsan_valloc(uptr size);
+void *nsan_pvalloc(uptr size);
+void *nsan_aligned_alloc(uptr alignment, uptr size);
+void *nsan_memalign(uptr alignment, uptr size);
+int nsan_posix_memalign(void **memptr, uptr alignment, uptr size);
+
+} // namespace __nsan
+#endif // NSAN_ALLOCATOR_H
diff --git a/compiler-rt/lib/nsan/nsan_flags.inc b/compiler-rt/lib/nsan/nsan_flags.inc
index 63c15475f6754bb..658cd5b3b01bf4d 100644
--- a/compiler-rt/lib/nsan/nsan_flags.inc
+++ b/compiler-rt/lib/nsan/nsan_flags.inc
@@ -46,4 +46,5 @@ NSAN_FLAG(bool, enable_loadtracking_stats, false,
           "If true, compute load tracking stats, i.e. for each load from "
           "memory, the number of times nsan resumed from the original value "
           "due to invalid or unknown types.")
+NSAN_FLAG(bool, poison_in_free, true, "")
 NSAN_FLAG(bool, print_stats_on_exit, false, "If true, print stats on exit.")
diff --git a/compiler-rt/lib/nsan/nsan_malloc_linux.cpp b/compiler-rt/lib/nsan/nsan_malloc_linux.cpp
index 02f52e7be07facb..c97591e4ac1593e 100644
--- a/compiler-rt/lib/nsan/nsan_malloc_linux.cpp
+++ b/compiler-rt/lib/nsan/nsan_malloc_linux.cpp
@@ -12,14 +12,16 @@
 
 #include "interception/interception.h"
 #include "nsan.h"
+#include "nsan_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_platform.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
 
 #if !SANITIZER_APPLE && !SANITIZER_WINDOWS
 using namespace __sanitizer;
-using __nsan::nsan_initialized;
+using namespace __nsan;
 
 namespace {
 struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
@@ -28,78 +30,53 @@ struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
 } // namespace
 
 INTERCEPTOR(void *, aligned_alloc, uptr align, uptr size) {
-  void *res = REAL(aligned_alloc)(align, size);
-  if (res)
-    __nsan_set_value_unknown(static_cast<u8 *>(res), size);
-  return res;
+  return nsan_aligned_alloc(align, size);
 }
 
 INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Callocate(nmemb, size);
-
-  void *res = REAL(calloc)(nmemb, size);
-  if (res)
-    __nsan_set_value_unknown(static_cast<u8 *>(res), nmemb * size);
-  return res;
+  return nsan_calloc(nmemb, size);
 }
 
 INTERCEPTOR(void, free, void *ptr) {
+  if (UNLIKELY(!ptr))
+    return;
   if (DlsymAlloc::PointerIsMine(ptr))
     return DlsymAlloc::Free(ptr);
-  REAL(free)(ptr);
+  NsanDeallocate(ptr);
 }
 
 INTERCEPTOR(void *, malloc, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Allocate(size);
-  void *res = REAL(malloc)(size);
-  if (res)
-    __nsan_set_value_unknown(static_cast<u8 *>(res), size);
-  return res;
+  return nsan_malloc(size);
 }
 
 INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
   if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
     return DlsymAlloc::Realloc(ptr, size);
-  void *res = REAL(realloc)(ptr, size);
-  // TODO: We might want to copy the types from the original allocation
-  // (although that would require that we know its size).
-  if (res)
-    __nsan_set_value_unknown(static_cast<u8 *>(res), size);
-  return res;
+  return nsan_realloc(ptr, size);
 }
 
 #if SANITIZER_INTERCEPT_REALLOCARRAY
 INTERCEPTOR(void *, reallocarray, void *ptr, uptr nmemb, uptr size) {
-  void *res = REAL(reallocarray)(ptr, nmemb, size);
-  if (res)
-    __nsan_set_value_unknown(static_cast<u8 *>(res), nmemb * size);
-  return res;
+  return nsan_reallocarray(ptr, nmemb, size);
 }
 #endif // SANITIZER_INTERCEPT_REALLOCARRAY
 
 INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr size) {
-  int res = REAL(posix_memalign)(memptr, align, size);
-  if (res == 0 && *memptr)
-    __nsan_set_value_unknown(static_cast<u8 *>(*memptr), size);
-  return res;
+  return nsan_posix_memalign(memptr, align, size);
 }
 
 // Deprecated allocation functions (memalign, etc).
 #if SANITIZER_INTERCEPT_MEMALIGN
 INTERCEPTOR(void *, memalign, uptr align, uptr size) {
-  void *const res = REAL(memalign)(align, size);
-  if (res)
-    __nsan_set_value_unknown(static_cast<u8 *>(res), size);
-  return res;
+  return nsan_memalign(align, size);
 }
 
 INTERCEPTOR(void *, __libc_memalign, uptr align, uptr size) {
-  void *const res = REAL(__libc_memalign)(align, size);
-  if (res)
-    __nsan_set_value_unknown(static_cast<u8 *>(res), size);
-  return res;
+  return nsan_memalign(align, size);
 }
 #endif
 
diff --git a/compiler-rt/lib/nsan/nsan_new_delete.cpp b/compiler-rt/lib/nsan/nsan_new_delete.cpp
new file mode 100644
index 000000000000000..f203a583f2c4480
--- /dev/null
+++ b/compiler-rt/lib/nsan/nsan_new_delete.cpp
@@ -0,0 +1,126 @@
+//===-- nsan_new_delete.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemorySanitizer.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+
+#include "interception/interception.h"
+#include "nsan.h"
+#include "nsan_allocator.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+
+#include <stddef.h>
+
+using namespace __nsan;
+
+// Fake std::nothrow_t and std::align_val_t to avoid including <new>.
+namespace std {
+struct nothrow_t {};
+enum class align_val_t : size_t {};
+} // namespace std
+
+#define OPERATOR_NEW_BODY(nothrow)                                             \
+  void *res = nsan_malloc(size);                                               \
+  if (!nothrow && UNLIKELY(!res)) {                                            \
+    BufferedStackTrace stack;                                                  \
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);                                    \
+    ReportOutOfMemory(size, &stack);                                           \
+  }                                                                            \
+  return res
+#define OPERATOR_NEW_BODY_ALIGN(nothrow)                                       \
+  void *res = nsan_memalign((uptr)align, size);                                \
+  if (!nothrow && UNLIKELY(!res)) {                                            \
+    BufferedStackTrace stack;                                                  \
+    GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);                                    \
+    ReportOutOfMemory(size, &stack);                                           \
+  }                                                                            \
+  return res;
+
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size) { OPERATOR_NEW_BODY(/*nothrow=*/false); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(/*nothrow=*/false); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::nothrow_t const &) {
+  OPERATOR_NEW_BODY(/*nothrow=*/true);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::nothrow_t const &) {
+  OPERATOR_NEW_BODY(/*nothrow=*/true);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN(/*nothrow=*/false);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN(/*nothrow=*/false);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align,
+                   std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ALIGN(/*nothrow=*/true);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align,
+                     std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ALIGN(/*nothrow=*/true);
+}
+
+#define OPERATOR_DELETE_BODY                                                   \
+  if (ptr)                                                                     \
+  NsanDeallocate(ptr)
+
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align,
+                     std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align,
+                       std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size,
+                       std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
diff --git a/compiler-rt/lib/nsan/nsan_platform.h b/compiler-rt/lib/nsan/nsan_platform.h
index c9d4cacd8c8898b..cc9d63ecb4a9baf 100644
--- a/compiler-rt/lib/nsan/nsan_platform.h
+++ b/compiler-rt/lib/nsan/nsan_platform.h
@@ -40,7 +40,9 @@ namespace __nsan {
 // |                    |
 // |       unused       |
 // |                    |
-// +--------------------+ 0x400000000000 (kUnusedAddr)
+// +--------------------+ 0x440000008000
+// |     allocator      |
+// +--------------------+ 0x400000000000 (kHeapMemBeg)
 // |   shadow memory    |
 // +--------------------+ 0x200000000000 (kShadowAddr)
 // |   shadow types     |
@@ -79,7 +81,7 @@ enum {
 struct Mapping {
   // FIXME: kAppAddr == 0x700000000000 ?
   static const uptr kAppAddr = 0x700000008000;
-  static const uptr kUnusedAddr = 0x400000000000;
+  static const uptr kHeapMemBeg = 0x400000000000;
   static const uptr kShadowAddr = 0x200000000000;
   static const uptr kTypesAddr = 0x100000000000;
   static const uptr kShadowMask = ~0x700000000000;
@@ -90,7 +92,7 @@ struct Mapping {
 
 enum MappingType {
   MAPPING_APP_ADDR,
-  MAPPING_UNUSED_ADDR,
+  MAPPING_ALLOCATOR_ADDR,
   MAPPING_SHADOW_ADDR,
   MAPPING_TYPES_ADDR,
   MAPPING_SHADOW_MASK
@@ -100,8 +102,8 @@ template <typename Mapping, int Type> uptr MappingImpl() {
   switch (Type) {
   case MAPPING_APP_ADDR:
     return Mapping::kAppAddr;
-  case MAPPING_UNUSED_ADDR:
-    return Mapping::kUnusedAddr;
+  case MAPPING_ALLOCATOR_ADDR:
+    return Mapping::kHeapMemBeg;
   case MAPPING_SHADOW_ADDR:
     return Mapping::kShadowAddr;
   case MAPPING_TYPES_ADDR:
@@ -119,7 +121,7 @@ ALWAYS_INLINE
 uptr AppAddr() { return MappingArchImpl<MAPPING_APP_ADDR>(); }
 
 ALWAYS_INLINE
-uptr UnusedAddr() { return MappingArchImpl<MAPPING_UNUSED_ADDR>(); }
+uptr AllocatorAddr() { return MappingArchImpl<MAPPING_ALLOCATOR_ADDR>(); }
 
 ALWAYS_INLINE
 uptr ShadowAddr() { return MappingArchImpl<MAPPING_SHADOW_ADDR>(); }
diff --git a/compiler-rt/lib/nsan/nsan_thread.cpp b/compiler-rt/lib/nsan/nsan_thread.cpp
index 273c46831cf3810..85706aea80ebd1a 100644
--- a/compiler-rt/lib/nsan/nsan_thread.cpp
+++ b/compiler-rt/lib/nsan/nsan_thread.cpp
@@ -55,6 +55,7 @@ void NsanThread::ClearShadowForThreadStackAndTLS() {
 void NsanThread::Init() {
   SetThreadStackAndTls();
   ClearShadowForThreadStackAndTLS();
+  malloc_storage().Init();
 }
 
 void NsanThread::TSDDtor(void *tsd) {
@@ -63,6 +64,7 @@ void NsanThread::TSDDtor(void *tsd) {
 }
 
 void NsanThread::Destroy() {
+  malloc_storage().CommitBack();
   // We also clear the shadow on thread destruction because
   // some code may still be executing in later TSD destructors
   // and we don't want it to have any poisoned stack.
diff --git a/compiler-rt/lib/nsan/nsan_thread.h b/compiler-rt/lib/nsan/nsan_thread.h
index 18f24fd6f1d78a9..143e61f37db96bb 100644
--- a/compiler-rt/lib/nsan/nsan_thread.h
+++ b/compiler-rt/lib/nsan/nsan_thread.h
@@ -9,6 +9,7 @@
 #ifndef NSAN_THREAD_H
 #define NSAN_THREAD_H
 
+#include "nsan_allocator.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_posix.h"
 
@@ -34,6 +35,8 @@ class NsanThread {
   void StartSwitchFiber(uptr bottom, uptr size);
   void FinishSwitchFiber(uptr *bottom_old, uptr *size_old);
 
+  NsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
+
   int destructor_iterations_;
   __sanitizer_sigset_t starting_sigset_;
 
@@ -56,6 +59,8 @@ class NsanThread {
 
   uptr tls_begin_;
   uptr tls_end_;
+
+  NsanThreadLocalMallocStorage malloc_storage_;
 };
 
 NsanThread *GetCurrentThread();
diff --git a/compiler-rt/lib/scudo/standalone/list.h b/compiler-rt/lib/scudo/standalone/list.h
index 0137667d1dcf3ef..6b952a610e30557 100644
--- a/compiler-rt/lib/scudo/standalone/list.h
+++ b/compiler-rt/lib/scudo/standalone/list.h
@@ -11,17 +11,113 @@
 
 #include "internal_defs.h"
 
+// TODO: Move the helpers to a header.
+namespace {
+template <typename T> struct isPointer {
+  static constexpr bool value = false;
+};
+
+template <typename T> struct isPointer<T *> {
+  static constexpr bool value = true;
+};
+} // namespace
+
 namespace scudo {
 
 // Intrusive POD singly and doubly linked list.
 // An object with all zero fields should represent a valid empty list. clear()
 // should be called on all non-zero-initialized objects before using.
+//
+// The intrusive list requires the member `Next` (and `Prev` if doubly linked
+// list)` defined in the node type. The type of `Next`/`Prev` can be a pointer
+// or an index to an array. For example, if the storage of the nodes is an
+// array, instead of using a pointer type, linking with an index type can save
+// some space.
+//
+// There are two things to be noticed while using an index type,
+//   1. Call init() to set up the base address of the array.
+//   2. Define `EndOfListVal` as the nil of the list.
+
+template <class T, bool LinkWithPtr = isPointer<decltype(T::Next)>::value>
+class LinkOp {
+public:
+  LinkOp() = default;
+  LinkOp(UNUSED T *BaseT, UNUSED uptr BaseSize) {}
+  void init(UNUSED T *LinkBase, UNUSED uptr Size) {}
+  T *getBase() const { return nullptr; }
+  uptr getSize() const { return 0; }
+
+  T *getNext(T *X) const { return X->Next; }
+  void setNext(T *X, T *Next) const { X->Next = Next; }
+
+  T *getPrev(T *X) const { return X->Prev; }
+  void setPrev(T *X, T *Prev) const { X->Prev = Prev; }
+
+  T *getEndOfListVal() const { return nullptr; }
+};
+
+template <class T> class LinkOp<T, /*LinkWithPtr=*/false> {
+public:
+  using LinkTy = decltype(T::Next);
+
+  LinkOp() = default;
+  LinkOp(T *BaseT, uptr BaseSize) : Base(BaseT), Size(BaseSize) {}
+  void init(T *LinkBase, uptr BaseSize) {
+    Base = LinkBase;
+    // TODO: Check if the `BaseSize` can fit in `Size`.
+    Size = static_cast<LinkTy>(BaseSize);
+  }
+  T *getBase() const { return Base; }
+  LinkTy getSize() const { return Size; }
+
+  T *getNext(T *X) const {
+    DCHECK_NE(getBase(), nullptr);
+    if (X->Next == getEndOfListVal())
+      return nullptr;
+    DCHECK_LT(X->Next, Size);
+    return &Base[X->Next];
+  }
+  // Set `X->Next` to `Next`.
+  void setNext(T *X, T *Next) const {
+    // TODO: Check if the offset fits in the size of `LinkTy`.
+    if (Next == nullptr)
+      X->Next = getEndOfListVal();
+    else
+      X->Next = static_cast<LinkTy>(Next - Base);
+  }
 
-template <class T> class IteratorBase {
+  T *getPrev(T *X) const {
+    DCHECK_NE(getBase(), nullptr);
+    if (X->Prev == getEndOfListVal())
+      return nullptr;
+    DCHECK_LT(X->Prev, Size);
+    return &Base[X->Prev];
+  }
+  // Set `X->Prev` to `Prev`.
+  void setPrev(T *X, T *Prev) const {
+    DCHECK_LT(reinterpret_cast<uptr>(Prev),
+              reinterpret_cast<uptr>(Base + Size));
+    if (Prev == nullptr)
+      X->Prev = getEndOfListVal();
+    else
+      X->Prev = static_cast<LinkTy>(Prev - Base);
+  }
+
+  // TODO: `LinkTy` should be the same as decltype(T::EndOfListVal).
+  LinkTy getEndOfListVal() const { return T::EndOfListVal; }
+
+protected:
+  T *Base = nullptr;
+  LinkTy Size = 0;
+};
+
+template <class T> class IteratorBase : public LinkOp<T> {
 public:
-  explicit IteratorBase(T *CurrentT) : Current(CurrentT) {}
+  IteratorBase(const LinkOp<T> &Link, T *CurrentT)
+      : LinkOp<T>(Link), Current(CurrentT) {}
+
   IteratorBase &operator++() {
-    Current = Current->Next;
+    Current = this->getNext(Current);
     return *this;
   }
   bool operator!=(IteratorBase Other) const { return Current != Other.Current; }
@@ -31,7 +127,10 @@ template <class T> class IteratorBase {
   T *Current;
 };
 
-template <class T> struct IntrusiveList {
+template <class T> struct IntrusiveList : public LinkOp<T> {
+  IntrusiveList() = default;
+  void init(T *Base, uptr BaseSize) { LinkOp<T>::init(Base, BaseSize); }
+
   bool empty() const { return Size == 0; }
   uptr size() const { return Size; }
 
@@ -48,11 +147,21 @@ template <class T> struct IntrusiveList {
   typedef IteratorBase<T> Iterator;
   typedef IteratorBase<const T> ConstIterator;
 
-  Iterator begin() { return Iterator(First); }
-  Iterator end() { return Iterator(nullptr); }
+  Iterator begin() {
+    return Iterator(LinkOp<T>(this->getBase(), this->getSize()), First);
+  }
+  Iterator end() {
+    return Iterator(LinkOp<T>(this->getBase(), this->getSize()), nullptr);
+  }
 
-  ConstIterator begin() const { return ConstIterator(First); }
-  ConstIterator end() const { return ConstIterator(nullptr); }
+  ConstIterator begin() const {
+    return ConstIterator(LinkOp<const T>(this->getBase(), this->getSize()),
+                         First);
+  }
+  ConstIterator end() const {
+    return ConstIterator(LinkOp<const T>(this->getBase(), this->getSize()),
+                         nullptr);
+  }
 
   void checkConsistency() const;
 
@@ -68,13 +177,13 @@ template <class T> void IntrusiveList<T>::checkConsistency() const {
     CHECK_EQ(Last, nullptr);
   } else {
     uptr Count = 0;
-    for (T *I = First;; I = I->Next) {
+    for (T *I = First;; I = this->getNext(I)) {
       Count++;
       if (I == Last)
         break;
     }
     CHECK_EQ(this->size(), Count);
-    CHECK_EQ(Last->Next, nullptr);
+    CHECK_EQ(this->getNext(Last), nullptr);
   }
 }
 
@@ -83,13 +192,16 @@ template <class T> struct SinglyLinkedList : public IntrusiveList<T> {
   using IntrusiveList<T>::Last;
   using IntrusiveList<T>::Size;
   using IntrusiveList<T>::empty;
+  using IntrusiveList<T>::setNext;
+  using IntrusiveList<T>::getNext;
+  using IntrusiveList<T>::getEndOfListVal;
 
   void push_back(T *X) {
-    X->Next = nullptr;
+    setNext(X, nullptr);
     if (empty())
       First = X;
     else
-      Last->Next = X;
+      setNext(Last, X);
     Last = X;
     Size++;
   }
@@ -97,14 +209,14 @@ template <class T> struct SinglyLinkedList : public IntrusiveList<T> {
   void push_front(T *X) {
     if (empty())
       Last = X;
-    X->Next = First;
+    setNext(X, First);
     First = X;
     Size++;
   }
 
   void pop_front() {
     DCHECK(!empty());
-    First = First->Next;
+    First = getNext(First);
     if (!First)
       Last = nullptr;
     Size--;
@@ -115,8 +227,8 @@ template <class T> struct SinglyLinkedList : public IntrusiveList<T> {
     DCHECK(!empty());
     DCHECK_NE(Prev, nullptr);
     DCHECK_NE(X, nullptr);
-    X->Next = Prev->Next;
-    Prev->Next = X;
+    setNext(X, getNext(Prev));
+    setNext(Prev, X);
     if (Last == Prev)
       Last = X;
     ++Size;
@@ -126,8 +238,8 @@ template <class T> struct SinglyLinkedList : public IntrusiveList<T> {
     DCHECK(!empty());
     DCHECK_NE(Prev, nullptr);
     DCHECK_NE(X, nullptr);
-    DCHECK_EQ(Prev->Next, X);
-    Prev->Next = X->Next;
+    DCHECK_EQ(getNext(Prev), X);
+    setNext(Prev, getNext(X));
     if (Last == X)
       Last = Prev;
     Size--;
@@ -140,7 +252,7 @@ template <class T> struct SinglyLinkedList : public IntrusiveList<T> {
     if (empty()) {
       *this = *L;
     } else {
-      Last->Next = L->First;
+      setNext(Last, L->First);
       Last = L->Last;
       Size += L->size();
     }
@@ -153,16 +265,21 @@ template <class T> struct DoublyLinkedList : IntrusiveList<T> {
   using IntrusiveList<T>::Last;
   using IntrusiveList<T>::Size;
   using IntrusiveList<T>::empty;
+  using IntrusiveList<T>::setNext;
+  using IntrusiveList<T>::getNext;
+  using IntrusiveList<T>::setPrev;
+  using IntrusiveList<T>::getPrev;
+  using IntrusiveList<T>::getEndOfListVal;
 
   void push_front(T *X) {
-    X->Prev = nullptr;
+    setPrev(X, nullptr);
     if (empty()) {
       Last = X;
     } else {
-      DCHECK_EQ(First->Prev, nullptr);
-      First->Prev = X;
+      DCHECK_EQ(getPrev(First), nullptr);
+      setPrev(First, X);
     }
-    X->Next = First;
+    setNext(X, First);
     First = X;
     Size++;
   }
@@ -171,37 +288,37 @@ template <class T> struct DoublyLinkedList : IntrusiveList<T> {
   void insert(T *X, T *Y) {
     if (Y == First)
       return push_front(X);
-    T *Prev = Y->Prev;
+    T *Prev = getPrev(Y);
     // This is a hard CHECK to ensure consistency in the event of an intentional
     // corruption of Y->Prev, to prevent a potential write-{4,8}.
-    CHECK_EQ(Prev->Next, Y);
-    Prev->Next = X;
-    X->Prev = Prev;
-    X->Next = Y;
-    Y->Prev = X;
+    CHECK_EQ(getNext(Prev), Y);
+    setNext(Prev, X);
+    setPrev(X, Prev);
+    setNext(X, Y);
+    setPrev(Y, X);
     Size++;
   }
 
   void push_back(T *X) {
-    X->Next = nullptr;
+    setNext(X, nullptr);
     if (empty()) {
       First = X;
     } else {
-      DCHECK_EQ(Last->Next, nullptr);
-      Last->Next = X;
+      DCHECK_EQ(getNext(Last), nullptr);
+      setNext(Last, X);
     }
-    X->Prev = Last;
+    setPrev(X, Last);
     Last = X;
     Size++;
   }
 
   void pop_front() {
     DCHECK(!empty());
-    First = First->Next;
+    First = getNext(First);
     if (!First)
       Last = nullptr;
     else
-      First->Prev = nullptr;
+      setPrev(First, nullptr);
     Size--;
   }
 
@@ -209,15 +326,15 @@ template <class T> struct DoublyLinkedList : IntrusiveList<T> {
   // catch potential corruption attempts, that could yield a mirrored
   // write-{4,8} primitive. nullptr checks are deemed less vital.
   void remove(T *X) {
-    T *Prev = X->Prev;
-    T *Next = X->Next;
+    T *Prev = getPrev(X);
+    T *Next = getNext(X);
     if (Prev) {
-      CHECK_EQ(Prev->Next, X);
-      Prev->Next = Next;
+      CHECK_EQ(getNext(Prev), X);
+      setNext(Prev, Next);
     }
     if (Next) {
-      CHECK_EQ(Next->Prev, X);
-      Next->Prev = Prev;
+      CHECK_EQ(getPrev(Next), X);
+      setPrev(Next, Prev);
     }
     if (First == X) {
       DCHECK_EQ(Prev, nullptr);
diff --git a/compiler-rt/lib/scudo/standalone/tests/list_test.cpp b/compiler-rt/lib/scudo/standalone/tests/list_test.cpp
index 140ca027ae92836..688cbbef6a032f2 100644
--- a/compiler-rt/lib/scudo/standalone/tests/list_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/list_test.cpp
@@ -10,25 +10,22 @@
 
 #include "list.h"
 
-struct ListItem {
-  ListItem *Next;
-  ListItem *Prev;
-};
+#include <array>
 
-static ListItem Items[6];
-static ListItem *X = &Items[0];
-static ListItem *Y = &Items[1];
-static ListItem *Z = &Items[2];
-static ListItem *A = &Items[3];
-static ListItem *B = &Items[4];
-static ListItem *C = &Items[5];
+struct ListItemLinkedWithPtr {
+  ListItemLinkedWithPtr *Next;
+  ListItemLinkedWithPtr *Prev;
+};
 
-typedef scudo::SinglyLinkedList<ListItem> SLList;
-typedef scudo::DoublyLinkedList<ListItem> DLList;
+struct ListItemLinkedWithIndex {
+  scudo::uptr Next;
+  scudo::uptr Prev;
+  static constexpr scudo::uptr EndOfListVal = 1ULL << 30;
+};
 
-template <typename ListT>
-static void setList(ListT *L, ListItem *I1 = nullptr, ListItem *I2 = nullptr,
-                    ListItem *I3 = nullptr) {
+template <typename ListT, typename ListItemTy>
+static void setList(ListT *L, ListItemTy *I1 = nullptr,
+                    ListItemTy *I2 = nullptr, ListItemTy *I3 = nullptr) {
   L->clear();
   if (I1)
     L->push_back(I1);
@@ -38,10 +35,10 @@ static void setList(ListT *L, ListItem *I1 = nullptr, ListItem *I2 = nullptr,
     L->push_back(I3);
 }
 
-template <typename ListT>
-static void checkList(ListT *L, ListItem *I1, ListItem *I2 = nullptr,
-                      ListItem *I3 = nullptr, ListItem *I4 = nullptr,
-                      ListItem *I5 = nullptr, ListItem *I6 = nullptr) {
+template <typename ListT, typename ListItemTy>
+static void checkList(ListT *L, ListItemTy *I1, ListItemTy *I2 = nullptr,
+                      ListItemTy *I3 = nullptr, ListItemTy *I4 = nullptr,
+                      ListItemTy *I5 = nullptr, ListItemTy *I6 = nullptr) {
   if (I1) {
     EXPECT_EQ(L->front(), I1);
     L->pop_front();
@@ -69,9 +66,16 @@ static void checkList(ListT *L, ListItem *I1, ListItem *I2 = nullptr,
   EXPECT_TRUE(L->empty());
 }
 
-template <typename ListT> static void testListCommon(void) {
-  ListT L;
+template <template <typename> class ListTy, typename ListItemTy>
+static void testListCommon(void) {
+  ListItemTy Items[3];
+  ListItemTy *X = &Items[0];
+  ListItemTy *Y = &Items[1];
+  ListItemTy *Z = &Items[2];
+
+  ListTy<ListItemTy> L;
   L.clear();
+  L.init(Items, sizeof(Items));
 
   EXPECT_EQ(L.size(), 0U);
   L.push_back(X);
@@ -123,16 +127,40 @@ template <typename ListT> static void testListCommon(void) {
   L.pop_front();
   EXPECT_TRUE(L.empty());
   L.checkConsistency();
+
+  L.push_back(X);
+  L.push_back(Y);
+  L.push_back(Z);
+
+  // Verify the iterator
+  std::array<ListItemTy *, 3> visitOrder{X, Y, Z};
+  auto Iter = visitOrder.begin();
+  for (const auto &Item : L) {
+    EXPECT_EQ(&Item, *Iter);
+    ++Iter;
+  }
 }
 
 TEST(ScudoListTest, LinkedListCommon) {
-  testListCommon<SLList>();
-  testListCommon<DLList>();
+  testListCommon<scudo::SinglyLinkedList, ListItemLinkedWithPtr>();
+  testListCommon<scudo::SinglyLinkedList, ListItemLinkedWithIndex>();
+  testListCommon<scudo::DoublyLinkedList, ListItemLinkedWithPtr>();
+  testListCommon<scudo::DoublyLinkedList, ListItemLinkedWithIndex>();
 }
 
-TEST(ScudoListTest, SinglyLinkedList) {
-  SLList L;
+template <template <typename> class ListTy, typename ListItemTy>
+static void testSinglyLinkedList() {
+  ListItemTy Items[6];
+  ListItemTy *X = &Items[0];
+  ListItemTy *Y = &Items[1];
+  ListItemTy *Z = &Items[2];
+  ListItemTy *A = &Items[3];
+  ListItemTy *B = &Items[4];
+  ListItemTy *C = &Items[5];
+
+  ListTy<ListItemTy> L;
   L.clear();
+  L.init(Items, sizeof(Items));
 
   L.push_back(X);
   L.push_back(Y);
@@ -150,9 +178,11 @@ TEST(ScudoListTest, SinglyLinkedList) {
   L.pop_front();
   EXPECT_TRUE(L.empty());
 
-  SLList L1, L2;
+  ListTy<ListItemTy> L1, L2;
   L1.clear();
   L2.clear();
+  L1.init(Items, sizeof(Items));
+  L2.init(Items, sizeof(Items));
 
   L1.append_back(&L2);
   EXPECT_TRUE(L1.empty());
@@ -180,9 +210,21 @@ TEST(ScudoListTest, SinglyLinkedList) {
   EXPECT_EQ(L1.size(), 1U);
 }
 
-TEST(ScudoListTest, DoublyLinkedList) {
-  DLList L;
+TEST(ScudoListTest, SinglyLinkedList) {
+  testSinglyLinkedList<scudo::SinglyLinkedList, ListItemLinkedWithPtr>();
+  testSinglyLinkedList<scudo::SinglyLinkedList, ListItemLinkedWithIndex>();
+}
+
+template <template <typename> class ListTy, typename ListItemTy>
+static void testDoublyLinkedList() {
+  ListItemTy Items[3];
+  ListItemTy *X = &Items[0];
+  ListItemTy *Y = &Items[1];
+  ListItemTy *Z = &Items[2];
+
+  ListTy<ListItemTy> L;
   L.clear();
+  L.init(Items, sizeof(Items));
 
   L.push_back(X);
   L.push_back(Y);
@@ -214,3 +256,8 @@ TEST(ScudoListTest, DoublyLinkedList) {
   L.pop_front();
   EXPECT_TRUE(L.empty());
 }
+
+TEST(ScudoListTest, DoublyLinkedList) {
+  testDoublyLinkedList<scudo::DoublyLinkedList, ListItemLinkedWithPtr>();
+  testDoublyLinkedList<scudo::DoublyLinkedList, ListItemLinkedWithIndex>();
+}
diff --git a/compiler-rt/test/nsan/Posix/allocator_mapping.cpp b/compiler-rt/test/nsan/Posix/allocator_mapping.cpp
new file mode 100644
index 000000000000000..3a3e655e259d0b2
--- /dev/null
+++ b/compiler-rt/test/nsan/Posix/allocator_mapping.cpp
@@ -0,0 +1,30 @@
+/// From msan/allocator_mapping.cpp
+/// Test that a module constructor can not map memory over the NSan heap
+/// (without MAP_FIXED, of course).
+// RUN: %clangxx_nsan -O0 %s -o %t_1
+// RUN: %clangxx_nsan -O0 -DHEAP_ADDRESS=$(%run %t_1) %s -o %t_2 && %run %t_2
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+
+#ifdef HEAP_ADDRESS
+struct A {
+  A() {
+    void *const hint = reinterpret_cast<void *>(HEAP_ADDRESS);
+    void *p = mmap(hint, 4096, PROT_READ | PROT_WRITE,
+                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    // This address must be already mapped. Check that mmap() succeeds, but at a
+    // different address.
+    assert(p != reinterpret_cast<void *>(-1));
+    assert(p != hint);
+  }
+} a;
+#endif
+
+int main() {
+  void *p = malloc(10);
+  printf("0x%zx\n", reinterpret_cast<size_t>(p) & (~0xfff));
+  free(p);
+}
diff --git a/compiler-rt/test/nsan/allocator_interface.cpp b/compiler-rt/test/nsan/allocator_interface.cpp
new file mode 100644
index 000000000000000..e658bd07291ad74
--- /dev/null
+++ b/compiler-rt/test/nsan/allocator_interface.cpp
@@ -0,0 +1,45 @@
+/// From sanitizer_common/TestCases/allocator_interface.cpp
+// RUN: %clangxx_nsan %s -o %t && %run %t 1234
+// RUN: %clangxx_nsan %s -o %t && %run %t 5678910
+
+#include <assert.h>
+#include <sanitizer/allocator_interface.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <thread>
+
+void Test(int size) {
+  auto allocated_bytes_before = __sanitizer_get_current_allocated_bytes();
+  int *p = (int *)malloc(size);
+  assert(__sanitizer_get_estimated_allocated_size(size) >= size);
+  assert(__sanitizer_get_ownership(p));
+  assert(!__sanitizer_get_ownership(&p));
+  assert(__sanitizer_get_allocated_size(p) == size);
+  assert(__sanitizer_get_allocated_size_fast(p) == size);
+  assert(__sanitizer_get_allocated_begin(p) == p);
+  assert(__sanitizer_get_allocated_begin(p + 1) == p);
+  assert(__sanitizer_get_current_allocated_bytes() >=
+         size + allocated_bytes_before);
+  assert(__sanitizer_get_current_allocated_bytes() <=
+         2 * size + allocated_bytes_before);
+  assert(__sanitizer_get_heap_size() >= size);
+  free(p);
+
+  // These are not implemented.
+  assert(__sanitizer_get_unmapped_bytes() <= 1);
+  assert(__sanitizer_get_free_bytes() > 0);
+
+  __sanitizer_purge_allocator();
+}
+
+int main(int argc, char **argv) {
+  int size = atoi(argv[1]);
+
+  Test(size);
+
+  // Check the thread local caches work as well.
+  std::thread t(Test, size);
+  t.join();
+
+  return 0;
+}
diff --git a/compiler-rt/test/nsan/malloc_hook.cpp b/compiler-rt/test/nsan/malloc_hook.cpp
new file mode 100644
index 000000000000000..3dcbfd845e75884
--- /dev/null
+++ b/compiler-rt/test/nsan/malloc_hook.cpp
@@ -0,0 +1,57 @@
+// RUN: %clangxx_nsan %s -o %t && %run %t 2>&1 | FileCheck %s
+
+#include <sanitizer/allocator_interface.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+extern "C" {
+const volatile void *global_ptr;
+
+#define WRITE(s) write(1, s, sizeof(s))
+
+// Note: avoid calling functions that allocate memory in malloc/free
+// to avoid infinite recursion.
+void __sanitizer_malloc_hook(const volatile void *ptr, size_t sz) {
+  if (__sanitizer_get_ownership(ptr) && sz == 4) {
+    WRITE("MallocHook\n");
+    global_ptr = ptr;
+  }
+}
+void __sanitizer_free_hook(const volatile void *ptr) {
+  if (__sanitizer_get_ownership(ptr) && ptr == global_ptr)
+    WRITE("FreeHook\n");
+}
+} // extern "C"
+
+volatile int *x;
+
+void MallocHook1(const volatile void *ptr, size_t sz) { WRITE("MH1\n"); }
+void MallocHook2(const volatile void *ptr, size_t sz) { WRITE("MH2\n"); }
+void FreeHook1(const volatile void *ptr) { WRITE("FH1\n"); }
+void FreeHook2(const volatile void *ptr) { WRITE("FH2\n"); }
+// Call this function with uninitialized arguments to poison
+// TLS shadow for function parameters before calling operator
+// new and, eventually, user-provided hook.
+__attribute__((noinline)) void allocate(int *unused1, int *unused2) {
+  x = new int;
+}
+
+int main() {
+  __sanitizer_install_malloc_and_free_hooks(MallocHook1, FreeHook1);
+  __sanitizer_install_malloc_and_free_hooks(MallocHook2, FreeHook2);
+  int *undef1, *undef2;
+  allocate(undef1, undef2);
+  // CHECK: MallocHook
+  // CHECK: MH1
+  // CHECK: MH2
+  // Check that malloc hook was called with correct argument.
+  if (global_ptr != (void *)x) {
+    _exit(1);
+  }
+  *x = 0;
+  delete x;
+  // CHECK: FreeHook
+  // CHECK: FH1
+  // CHECK: FH2
+  return 0;
+}
diff --git a/compiler-rt/test/nsan/new_delete_test.cpp b/compiler-rt/test/nsan/new_delete_test.cpp
new file mode 100644
index 000000000000000..356145e9bf21f02
--- /dev/null
+++ b/compiler-rt/test/nsan/new_delete_test.cpp
@@ -0,0 +1,77 @@
+/// From sanitizer_common/TestCases/Linux/new_delete_test.cpp
+// RUN: %clangxx_nsan -fno-sized-deallocation -O0 %s -o %t && %run %t
+// RUN: %clangxx_nsan -fsized-deallocation -O0 %s -o %t && %run %t
+
+#include <cstddef>
+
+namespace std {
+struct nothrow_t {};
+static const nothrow_t nothrow;
+enum class align_val_t : size_t {};
+} // namespace std
+
+void *operator new(size_t);
+void *operator new[](size_t);
+void *operator new(size_t, std::nothrow_t const &);
+void *operator new[](size_t, std::nothrow_t const &);
+void *operator new(size_t, std::align_val_t);
+void *operator new[](size_t, std::align_val_t);
+void *operator new(size_t, std::align_val_t, std::nothrow_t const &);
+void *operator new[](size_t, std::align_val_t, std::nothrow_t const &);
+
+void operator delete(void *) throw();
+void operator delete[](void *) throw();
+void operator delete(void *, std::nothrow_t const &);
+void operator delete[](void *, std::nothrow_t const &);
+void operator delete(void *, size_t) throw();
+void operator delete[](void *, size_t) throw();
+void operator delete(void *, std::align_val_t) throw();
+void operator delete[](void *, std::align_val_t) throw();
+void operator delete(void *, std::align_val_t, std::nothrow_t const &);
+void operator delete[](void *, std::align_val_t, std::nothrow_t const &);
+void operator delete(void *, size_t, std::align_val_t) throw();
+void operator delete[](void *, size_t, std::align_val_t) throw();
+
+template <typename T> inline T *break_optimization(T *arg) {
+  __asm__ __volatile__("" : : "r"(arg) : "memory");
+  return arg;
+}
+
+struct S12 {
+  int a, b, c;
+};
+struct alignas(128) S12_128 {
+  int a, b, c;
+};
+struct alignas(256) S12_256 {
+  int a, b, c;
+};
+struct alignas(512) S1024_512 {
+  char a[1024];
+};
+struct alignas(1024) S1024_1024 {
+  char a[1024];
+};
+
+int main(int argc, char **argv) {
+  delete break_optimization(new S12);
+  operator delete(break_optimization(new S12), std::nothrow);
+  delete[] break_optimization(new S12[100]);
+  operator delete[](break_optimization(new S12[100]), std::nothrow);
+
+  delete break_optimization(new S12_128);
+  operator delete(break_optimization(new S12_128),
+                  std::align_val_t(alignof(S12_128)));
+  operator delete(break_optimization(new S12_128),
+                  std::align_val_t(alignof(S12_128)), std::nothrow);
+  operator delete(break_optimization(new S12_128), sizeof(S12_128),
+                  std::align_val_t(alignof(S12_128)));
+
+  delete[] break_optimization(new S12_128[100]);
+  operator delete[](break_optimization(new S12_128[100]),
+                    std::align_val_t(alignof(S12_128)));
+  operator delete[](break_optimization(new S12_128[100]),
+                    std::align_val_t(alignof(S12_128)), std::nothrow);
+  operator delete[](break_optimization(new S12_128[100]), sizeof(S12_128[100]),
+                    std::align_val_t(alignof(S12_128)));
+}
diff --git a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
index bd31aa0782493c4..e45011c8e02a33b 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/FIRContext.h
@@ -71,6 +71,12 @@ void setTargetFeatures(mlir::ModuleOp mod, llvm::StringRef features);
 /// Get the target features from the Module.
 mlir::LLVM::TargetFeaturesAttr getTargetFeatures(mlir::ModuleOp mod);
 
+/// Set the compiler identifier for the module.
+void setIdent(mlir::ModuleOp mod, llvm::StringRef ident);
+
+/// Get the compiler identifier from the Module.
+llvm::StringRef getIdent(mlir::ModuleOp mod);
+
 /// Helper for determining the target from the host, etc. Tools may use this
 /// function to provide a consistent interpretation of the `--target=<string>`
 /// command-line option.
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 3d5b2e2a2fe028a..ccbb481f472d819 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Lower/Bridge.h"
+#include "flang/Common/Version.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
@@ -6125,6 +6126,7 @@ Fortran::lower::LoweringBridge::LoweringBridge(
   fir::setTargetFeatures(*module.get(), targetMachine.getTargetFeatureString());
   fir::support::setMLIRDataLayout(*module.get(),
                                   targetMachine.createDataLayout());
+  fir::setIdent(*module.get(), Fortran::common::getFlangFullVersion());
 }
 
 void Fortran::lower::genCleanUpInRegionIfAny(
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 14723360f0ee1ce..1c0e541e4a36a79 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2069,10 +2069,12 @@ static void genCompositeDistributeSimd(
   // TODO: Populate entry block arguments with private variables.
   auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
       converter, loc, distributeClauseOps, /*blockArgTypes=*/{});
+  distributeOp.setComposite(/*val=*/true);
 
   // TODO: Populate entry block arguments with reduction and private variables.
   auto simdOp = genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps,
                                                 /*blockArgTypes=*/{});
+  simdOp.setComposite(/*val=*/true);
 
   // Construct wrapper entry block list and associated symbols. It is important
   // that the symbol order and the block argument order match, so that the
@@ -2117,10 +2119,12 @@ static void genCompositeDoSimd(lower::AbstractConverter &converter,
   // TODO: Add private variables to entry block arguments.
   auto wsloopOp = genWrapperOp<mlir::omp::WsloopOp>(
       converter, loc, wsloopClauseOps, wsloopReductionTypes);
+  wsloopOp.setComposite(/*val=*/true);
 
   // TODO: Populate entry block arguments with reduction and private variables.
   auto simdOp = genWrapperOp<mlir::omp::SimdOp>(converter, loc, simdClauseOps,
                                                 /*blockArgTypes=*/{});
+  simdOp.setComposite(/*val=*/true);
 
   // Construct wrapper entry block list and associated symbols. It is important
   // that the symbol and block argument order match, so that the symbol-value
diff --git a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
index 1aa631cb3912690..5bd8e2d43363616 100644
--- a/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/FIRContext.cpp
@@ -114,6 +114,22 @@ mlir::LLVM::TargetFeaturesAttr fir::getTargetFeatures(mlir::ModuleOp mod) {
   return {};
 }
 
+void fir::setIdent(mlir::ModuleOp mod, llvm::StringRef ident) {
+  if (ident.empty())
+    return;
+
+  mlir::MLIRContext *ctx = mod.getContext();
+  mod->setAttr(mlir::LLVM::LLVMDialect::getIdentAttrName(),
+               mlir::StringAttr::get(ctx, ident));
+}
+
+llvm::StringRef fir::getIdent(mlir::ModuleOp mod) {
+  if (auto attr = mod->getAttrOfType<mlir::StringAttr>(
+          mlir::LLVM::LLVMDialect::getIdentAttrName()))
+    return attr;
+  return {};
+}
+
 std::string fir::determineTargetTriple(llvm::StringRef triple) {
   // Treat "" or "default" as stand-ins for the default machine.
   if (triple.empty() || triple == "default")
diff --git a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
index 7d0b8b3d7bc53d5..1e44288c784c2a9 100644
--- a/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
+++ b/flang/lib/Optimizer/Transforms/ConstantArgumentGlobalisation.cpp
@@ -98,9 +98,8 @@ class CallOpRewriter : public mlir::OpRewritePattern<fir::CallOp> {
       assert(!builder.getNamedGlobal(globalName) &&
              "We should have a unique name here");
 
-      if (std::find_if(allocas.begin(), allocas.end(), [alloca](auto x) {
-            return x.first == alloca;
-          }) == allocas.end()) {
+      if (llvm::none_of(allocas,
+                        [alloca](auto x) { return x.first == alloca; })) {
         allocas.push_back(std::make_pair(alloca, store));
       }
 
diff --git a/flang/lib/Semantics/data-to-inits.cpp b/flang/lib/Semantics/data-to-inits.cpp
index b6dbbbf63138ec0..0f8b36de460865a 100644
--- a/flang/lib/Semantics/data-to-inits.cpp
+++ b/flang/lib/Semantics/data-to-inits.cpp
@@ -522,11 +522,10 @@ static const DerivedTypeSpec *HasDefaultInitialization(const Symbol &symbol) {
     } else if (!object->isDummy() && object->type()) {
       if (const DerivedTypeSpec * derived{object->type()->AsDerived()}) {
         DirectComponentIterator directs{*derived};
-        if (std::find_if(
-                directs.begin(), directs.end(), [](const Symbol &component) {
-                  return !IsAllocatable(component) &&
-                      HasDeclarationInitializer(component);
-                }) != directs.end()) {
+        if (llvm::any_of(directs, [](const Symbol &component) {
+              return !IsAllocatable(component) &&
+                  HasDeclarationInitializer(component);
+            })) {
           return derived;
         }
       }
diff --git a/flang/lib/Semantics/rewrite-directives.cpp b/flang/lib/Semantics/rewrite-directives.cpp
index 2c3c87f2546a356..c94d0f3855bee31 100644
--- a/flang/lib/Semantics/rewrite-directives.cpp
+++ b/flang/lib/Semantics/rewrite-directives.cpp
@@ -88,11 +88,9 @@ bool OmpRewriteMutator::Pre(parser::OpenMPAtomicConstruct &x) {
 
   auto findMemOrderClause =
       [](const std::list<parser::OmpAtomicClause> &clauses) {
-        return std::find_if(
-                   clauses.begin(), clauses.end(), [](const auto &clause) {
-                     return std::get_if<parser::OmpMemoryOrderClause>(
-                         &clause.u);
-                   }) != clauses.end();
+        return llvm::any_of(clauses, [](const auto &clause) {
+          return std::get_if<parser::OmpMemoryOrderClause>(&clause.u);
+        });
       };
 
   // Get the clause list to which the new memory order clause must be added,
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 31e91ee7355e332..b593bf89b18bc94 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -230,11 +230,10 @@ void GenericDetails::CopyFrom(const GenericDetails &from) {
     derivedType_ = from.derivedType_;
   }
   for (std::size_t i{0}; i < from.specificProcs_.size(); ++i) {
-    if (std::find_if(specificProcs_.begin(), specificProcs_.end(),
-            [&](const Symbol &mySymbol) {
-              return &mySymbol.GetUltimate() ==
-                  &from.specificProcs_[i]->GetUltimate();
-            }) == specificProcs_.end()) {
+    if (llvm::none_of(specificProcs_, [&](const Symbol &mySymbol) {
+          return &mySymbol.GetUltimate() ==
+              &from.specificProcs_[i]->GetUltimate();
+        })) {
       specificProcs_.push_back(from.specificProcs_[i]);
       bindingNames_.push_back(from.bindingNames_[i]);
     }
diff --git a/flang/test/Lower/ident.f90 b/flang/test/Lower/ident.f90
new file mode 100644
index 000000000000000..c5201edbcf83f7b
--- /dev/null
+++ b/flang/test/Lower/ident.f90
@@ -0,0 +1,5 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: module attributes {
+! CHECK-SAME: llvm.ident = "flang version {{.+}}"
diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index 5e05c1617a3be01..b2644d2ebf386cf 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -251,16 +251,26 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.atanf
     libc.src.math.atanh
     libc.src.math.atanhf
+    libc.src.math.canonicalize
+    libc.src.math.canonicalizef
+    libc.src.math.canonicalizel
     libc.src.math.cbrt
     libc.src.math.cbrtf
     libc.src.math.ceil
     libc.src.math.ceilf
+    libc.src.math.ceill
     libc.src.math.copysign
     libc.src.math.copysignf
+    libc.src.math.copysignl
     libc.src.math.cos
     libc.src.math.cosf
     libc.src.math.cosh
     libc.src.math.coshf
+    libc.src.math.cospif
+    libc.src.math.ddivl
+    libc.src.math.dfmal
+    libc.src.math.dmull
+    libc.src.math.dsqrtl
     libc.src.math.erf
     libc.src.math.erff
     libc.src.math.exp
@@ -268,40 +278,101 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.exp10f
     libc.src.math.exp2
     libc.src.math.exp2f
+    libc.src.math.exp2m1f
     libc.src.math.expf
     libc.src.math.expm1
     libc.src.math.expm1f
     libc.src.math.fabs
     libc.src.math.fabsf
+    libc.src.math.fabsl
     libc.src.math.fadd
+    libc.src.math.faddl
     libc.src.math.fdim
     libc.src.math.fdimf
+    libc.src.math.fdiml
+    libc.src.math.fdiv
+    libc.src.math.fdivl
+    libc.src.math.ffma
+    libc.src.math.ffmal
     libc.src.math.floor
     libc.src.math.floorf
+    libc.src.math.floorl
     libc.src.math.fma
     libc.src.math.fmaf
     libc.src.math.fmax
     libc.src.math.fmaxf
+    libc.src.math.fmaximum
+    libc.src.math.fmaximumf
+    libc.src.math.fmaximuml
+    libc.src.math.fmaximum_mag
+    libc.src.math.fmaximum_magf
+    libc.src.math.fmaximum_magl
+    libc.src.math.fmaximum_mag_num
+    libc.src.math.fmaximum_mag_numf
+    libc.src.math.fmaximum_mag_numl
+    libc.src.math.fmaximum_num
+    libc.src.math.fmaximum_numf
+    libc.src.math.fmaximum_numl
+    libc.src.math.fmaxl
     libc.src.math.fmin
     libc.src.math.fminf
+    libc.src.math.fminimum
+    libc.src.math.fminimumf
+    libc.src.math.fminimuml
+    libc.src.math.fminimum_mag
+    libc.src.math.fminimum_magf
+    libc.src.math.fminimum_magl
+    libc.src.math.fminimum_mag_num
+    libc.src.math.fminimum_mag_numf
+    libc.src.math.fminimum_mag_numl
+    libc.src.math.fminimum_num
+    libc.src.math.fminimum_numf
+    libc.src.math.fminimum_numl
+    libc.src.math.fminl
     libc.src.math.fmod
     libc.src.math.fmodf
+    libc.src.math.fmodl
+    libc.src.math.fmul
+    libc.src.math.fmull
     libc.src.math.frexp
     libc.src.math.frexpf
+    libc.src.math.frexpl
+    # FIXME: Broken on NVPTX.
+    # libc.src.math.fromfp
+    # libc.src.math.fromfpf
+    # libc.src.math.fromfpl
+    # libc.src.math.fromfpx
+    # libc.src.math.fromfpxf
+    # libc.src.math.fromfpxl
+    libc.src.math.fsqrt
+    libc.src.math.fsqrtl
+    libc.src.math.fsub
+    libc.src.math.fsubl
     libc.src.math.getpayload
     libc.src.math.getpayloadf
+    libc.src.math.getpayloadl
     libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
     libc.src.math.ilogbf
+    libc.src.math.ilogbl
+    libc.src.math.isnan
+    libc.src.math.isnanf
+    libc.src.math.isnanl
     libc.src.math.ldexp
     libc.src.math.ldexpf
+    libc.src.math.ldexpl
+    libc.src.math.lgamma
+    libc.src.math.lgamma_r
     libc.src.math.llogb
     libc.src.math.llogbf
+    libc.src.math.llogbl
     libc.src.math.llrint
     libc.src.math.llrintf
+    libc.src.math.llrintl
     libc.src.math.llround
     libc.src.math.llroundf
+    libc.src.math.llroundl
     libc.src.math.log
     libc.src.math.log10
     libc.src.math.log10f
@@ -309,55 +380,97 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.log1pf
     libc.src.math.log2
     libc.src.math.log2f
+    libc.src.math.logb
+    libc.src.math.logbf
+    libc.src.math.logbl
     libc.src.math.logf
     libc.src.math.lrint
     libc.src.math.lrintf
+    libc.src.math.lrintl
     libc.src.math.lround
     libc.src.math.lroundf
+    libc.src.math.lroundl
     libc.src.math.modf
     libc.src.math.modff
+    libc.src.math.modfl
     libc.src.math.nan
     libc.src.math.nanf
+    libc.src.math.nanl
     libc.src.math.nearbyint
     libc.src.math.nearbyintf
+    libc.src.math.nearbyintl
     libc.src.math.nextafter
     libc.src.math.nextafterf
+    libc.src.math.nextafterl
+    libc.src.math.nextdown
+    libc.src.math.nextdownf
+    libc.src.math.nextdownl
     libc.src.math.nexttoward
     libc.src.math.nexttowardf
+    libc.src.math.nexttowardl
+    libc.src.math.nextup
+    libc.src.math.nextupf
+    libc.src.math.nextupl
     libc.src.math.pow
     libc.src.math.powf
     libc.src.math.powi
     libc.src.math.powif
     libc.src.math.remainder
     libc.src.math.remainderf
+    libc.src.math.remainderl
     libc.src.math.remquo
     libc.src.math.remquof
+    libc.src.math.remquol
     libc.src.math.rint
     libc.src.math.rintf
+    libc.src.math.rintl
     libc.src.math.round
+    libc.src.math.roundeven
+    libc.src.math.roundevenf
+    libc.src.math.roundevenl
     libc.src.math.roundf
     libc.src.math.scalbln
     libc.src.math.scalblnf
     libc.src.math.scalbn
     libc.src.math.scalbnf
+    libc.src.math.scalbnl
+    libc.src.math.setpayload
+    libc.src.math.setpayloadf
+    libc.src.math.setpayloadl
+    libc.src.math.setpayloadsig
+    libc.src.math.setpayloadsigf
+    libc.src.math.setpayloadsigl
     libc.src.math.sin
     libc.src.math.sincos
     libc.src.math.sincosf
     libc.src.math.sinf
     libc.src.math.sinh
     libc.src.math.sinhf
+    libc.src.math.sinpif
     libc.src.math.sqrt
     libc.src.math.sqrtf
+    libc.src.math.sqrtl
     libc.src.math.tan
     libc.src.math.tanf
     libc.src.math.tanh
     libc.src.math.tanhf
     libc.src.math.tgamma
     libc.src.math.tgammaf
-    libc.src.math.lgamma
-    libc.src.math.lgamma_r
+    libc.src.math.totalorder
+    libc.src.math.totalorderf
+    libc.src.math.totalordermag
+    libc.src.math.totalordermagf
+    libc.src.math.totalordermagl
     libc.src.math.trunc
     libc.src.math.truncf
+    libc.src.math.truncl
+    # FIXME: Broken on NVPTX.
+    # libc.src.math.ufromfp
+    # libc.src.math.ufromfpf
+    # libc.src.math.ufromfpl
+    # libc.src.math.ufromfpx
+    # libc.src.math.ufromfpxf
+    # libc.src.math.ufromfpxl
 )
 
 if(LIBC_TYPES_HAS_FLOAT16)
@@ -366,18 +479,27 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.canonicalizef16
     libc.src.math.ceilf16
     libc.src.math.copysignf16
+    libc.src.math.exp10f16
+    libc.src.math.exp2f16
+    libc.src.math.expf16
     libc.src.math.f16add
     libc.src.math.f16addf
+    libc.src.math.f16addl
     libc.src.math.f16div
     libc.src.math.f16divf
+    libc.src.math.f16divl
     libc.src.math.f16fma
     libc.src.math.f16fmaf
+    libc.src.math.f16fmal
     libc.src.math.f16mul
     libc.src.math.f16mulf
+    libc.src.math.f16mull
     libc.src.math.f16sqrt
     libc.src.math.f16sqrtf
+    libc.src.math.f16sqrtl
     libc.src.math.f16sub
     libc.src.math.f16subf
+    libc.src.math.f16subl
     libc.src.math.fabsf16
     libc.src.math.fdimf16
     libc.src.math.floorf16
diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h
index 3bcfc190026257d..ffd6ebf27c77264 100644
--- a/libc/src/__support/str_to_float.h
+++ b/libc/src/__support/str_to_float.h
@@ -1159,13 +1159,11 @@ LIBC_INLINE StrToNumResult<T> strtofloatingpoint(const char *__restrict src) {
       index += 3;
       StorageType nan_mantissa = 0;
       // this handles the case of `NaN(n-character-sequence)`, where the
-      // n-character-sequence is made of 0 or more letters and numbers in any
-      // order.
+      // n-character-sequence is made of 0 or more letters, numbers, or
+      // underscore characters in any order.
       if (src[index] == '(') {
         size_t left_paren = index;
         ++index;
-        // Apparently it's common for underscores to also be accepted. No idea
-        // why, but it's causing fuzz failures.
         while (isalnum(src[index]) || src[index] == '_')
           ++index;
         if (src[index] == ')') {
diff --git a/libc/test/UnitTest/LibcTestMain.cpp b/libc/test/UnitTest/LibcTestMain.cpp
index 94536e971646865..c348d5ef1aa1b12 100644
--- a/libc/test/UnitTest/LibcTestMain.cpp
+++ b/libc/test/UnitTest/LibcTestMain.cpp
@@ -43,7 +43,15 @@ TestOptions parseOptions(int argc, char **argv) {
 
 } // anonymous namespace
 
-extern "C" int main(int argc, char **argv, char **envp) {
+// The C++ standard forbids declaring the main function with a linkage specifier
+// outisde of 'freestanding' mode, only define the linkage for hermetic tests.
+#if __STDC_HOSTED__
+#define TEST_MAIN int main
+#else
+#define TEST_MAIN extern "C" int main
+#endif
+
+TEST_MAIN(int argc, char **argv, char **envp) {
   LIBC_NAMESPACE::testing::argc = argc;
   LIBC_NAMESPACE::testing::argv = argv;
   LIBC_NAMESPACE::testing::envp = envp;
diff --git a/libc/test/src/stdlib/strtof_test.cpp b/libc/test/src/stdlib/strtof_test.cpp
index d7991745b69e6cd..6a716c956291cc2 100644
--- a/libc/test/src/stdlib/strtof_test.cpp
+++ b/libc/test/src/stdlib/strtof_test.cpp
@@ -200,7 +200,7 @@ TEST_F(LlvmLibcStrToFTest, NaNWithParenthesesValidSequenceInvalidNumberTests) {
   run_test("NaN(1a)", 7, 0x7fc00000);
   run_test("NaN(asdf)", 9, 0x7fc00000);
   run_test("NaN(1A1)", 8, 0x7fc00000);
-  run_test("NaN(why_does_this_work)", 23, 0x7fc00000);
+  run_test("NaN(underscores_are_ok)", 23, 0x7fc00000);
   run_test(
       "NaN(1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM_)",
       68, 0x7fc00000);
diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index c4c6a855cc517be..b6381f86323dbd4 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -159,7 +159,7 @@
 "`LWG3660 <https://wg21.link/LWG3660>`__","``iterator_traits<common_iterator>::pointer`` should conform to §[iterator.traits]","February 2022","|Complete|","14.0","|ranges|"
 "`LWG3661 <https://wg21.link/LWG3661>`__","``constinit atomic<shared_ptr<T>> a(nullptr);`` should work","February 2022","","",""
 "","","","","",""
-"`LWG3564 <https://wg21.link/LWG3564>`__","``transform_view::iterator<true>::value_type`` and ``iterator_category`` should use ``const F&``","July 2022","","","|ranges|"
+"`LWG3564 <https://wg21.link/LWG3564>`__","``transform_view::iterator<true>::value_type`` and ``iterator_category`` should use ``const F&``","July 2022","|Complete|","20.0","|ranges|"
 "`LWG3617 <https://wg21.link/LWG3617>`__","``function``/``packaged_task`` deduction guides and deducing ``this``","July 2022","","",""
 "`LWG3656 <https://wg21.link/LWG3656>`__","Inconsistent bit operations returning a count","July 2022","|Complete|","15.0",""
 "`LWG3659 <https://wg21.link/LWG3659>`__","Consider ``ATOMIC_FLAG_INIT`` undeprecation","July 2022","|Complete|","15.0",""
diff --git a/libcxx/include/__ranges/transform_view.h b/libcxx/include/__ranges/transform_view.h
index bcce389c0e6809f..686a2a8fe8c21e3 100644
--- a/libcxx/include/__ranges/transform_view.h
+++ b/libcxx/include/__ranges/transform_view.h
@@ -173,7 +173,8 @@ template <input_range _View, copy_constructible _Fn>
 #  endif
   requires __transform_view_constraints<_View, _Fn>
 template <bool _Const>
-class transform_view<_View, _Fn>::__iterator : public __transform_view_iterator_category_base<_View, _Fn> {
+class transform_view<_View, _Fn>::__iterator
+    : public __transform_view_iterator_category_base<_View, __maybe_const<_Const, _Fn>> {
 
   using _Parent = __maybe_const<_Const, transform_view>;
   using _Base   = __maybe_const<_Const, _View>;
@@ -190,7 +191,7 @@ class transform_view<_View, _Fn>::__iterator : public __transform_view_iterator_
   iterator_t<_Base> __current_ = iterator_t<_Base>();
 
   using iterator_concept = typename __transform_view_iterator_concept<_View>::type;
-  using value_type       = remove_cvref_t<invoke_result_t<_Fn&, range_reference_t<_Base>>>;
+  using value_type       = remove_cvref_t<invoke_result_t<__maybe_const<_Const, _Fn>&, range_reference_t<_Base>>>;
   using difference_type  = range_difference_t<_Base>;
 
   _LIBCPP_HIDE_FROM_ABI __iterator()
diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
index 4c2b483914f47e0..2c8534977febc7e 100644
--- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
+++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
@@ -20,6 +20,9 @@
 // support gdb anymore, favoring lldb instead.
 // UNSUPPORTED: android
 
+// This test doesn't work as such on Windows.
+// UNSUPPORTED: windows
+
 // RUN: %{cxx} %{flags} %s -o %t.exe %{compile_flags} -g %{link_flags}
 // Ensure locale-independence for unicode tests.
 // RUN: env LANG=en_US.UTF-8 %{gdb} -nx -batch -iex "set autoload off" -ex "source %S/../../../utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %S/gdb_pretty_printer_test.py" %t.exe
diff --git a/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/types.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/types.pass.cpp
index 25d8936f4e4ebd1..607b748ac3e4bd8 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/types.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.transform/iterator/types.pass.cpp
@@ -61,6 +61,22 @@ constexpr bool test() {
     static_assert(std::same_as<typename TIter::value_type, int>);
     static_assert(std::same_as<typename TIter::difference_type, std::ptrdiff_t>);
   }
+  {
+    // Member typedefs for random access iterator, LWG3564 mutable overload
+    using TView = std::ranges::transform_view<RandomAccessView, PlusOneMutableOverload>;
+    using TIter = std::ranges::iterator_t<TView>;
+    static_assert(std::same_as<typename TIter::iterator_concept, std::random_access_iterator_tag>);
+    static_assert(std::same_as<typename TIter::iterator_category, std::random_access_iterator_tag>);
+    static_assert(std::same_as<typename TIter::value_type, int>);
+    static_assert(std::same_as<typename TIter::difference_type, std::ptrdiff_t>);
+
+    using CTIter = std::ranges::iterator_t<const TView>;
+    static_assert(std::same_as<typename CTIter::iterator_concept, std::random_access_iterator_tag>);
+    static_assert(std::same_as<typename CTIter::iterator_category,
+                               std::input_iterator_tag>); // Note: this is now input_iterator_tag.
+    static_assert(std::same_as<typename CTIter::value_type, int>);
+    static_assert(std::same_as<typename CTIter::difference_type, std::ptrdiff_t>);
+  }
   {
     // Member typedefs for bidirectional iterator.
     using TView = std::ranges::transform_view<BidirectionalView, Increment>;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.transform/types.h b/libcxx/test/std/ranges/range.adaptors/range.transform/types.h
index cc5679f229de250..e94e7dfcdaeb7e9 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.transform/types.h
+++ b/libcxx/test/std/ranges/range.adaptors/range.transform/types.h
@@ -131,6 +131,11 @@ struct PlusOne {
   constexpr int operator()(int x) const { return x + 1; }
 };
 
+struct PlusOneMutableOverload {
+  constexpr int operator()(int x) const { return x + 1; }
+  constexpr int& operator()(int& x) { return ++x; }
+};
+
 struct Increment {
   constexpr int& operator()(int& x) { return ++x; }
 };
diff --git a/libcxx/test/support/count_new.h b/libcxx/test/support/count_new.h
index 0a95e05b72421bc..2298c4fd63e845b 100644
--- a/libcxx/test/support/count_new.h
+++ b/libcxx/test/support/count_new.h
@@ -455,7 +455,7 @@ void operator delete[](void* p, std::nothrow_t const&) TEST_NOEXCEPT {
 #      define USE_ALIGNED_ALLOC
 #    endif
 
-inline void* alocate_aligned_impl(std::size_t size, std::align_val_t align) {
+inline void* allocate_aligned_impl(std::size_t size, std::align_val_t align) {
   const std::size_t alignment = static_cast<std::size_t>(align);
   void* ret                   = nullptr;
 #    ifdef USE_ALIGNED_ALLOC
@@ -479,7 +479,7 @@ inline void free_aligned_impl(void* ptr, std::align_val_t) {
 // operator new(size_t, align_val_t[, nothrow_t]) and operator delete(size_t, align_val_t[, nothrow_t])
 void* operator new(std::size_t s, std::align_val_t av) TEST_THROW_SPEC(std::bad_alloc) {
   getGlobalMemCounter()->alignedNewCalled(s, static_cast<std::size_t>(av));
-  void* p = alocate_aligned_impl(s, av);
+  void* p = allocate_aligned_impl(s, av);
   if (p == nullptr)
     detail::throw_bad_alloc_helper();
   return p;
@@ -495,7 +495,7 @@ void* operator new(std::size_t s, std::align_val_t av, std::nothrow_t const&) TE
     return nullptr;
   }
 #    endif
-  return alocate_aligned_impl(s, av);
+  return allocate_aligned_impl(s, av);
 }
 
 void operator delete(void* p, std::align_val_t av) TEST_NOEXCEPT {
@@ -511,7 +511,7 @@ void operator delete(void* p, std::align_val_t av, std::nothrow_t const&) TEST_N
 // operator new[](size_t, align_val_t[, nothrow_t]) and operator delete[](size_t, align_val_t[, nothrow_t])
 void* operator new[](std::size_t s, std::align_val_t av) TEST_THROW_SPEC(std::bad_alloc) {
   getGlobalMemCounter()->alignedNewArrayCalled(s, static_cast<std::size_t>(av));
-  void* p = alocate_aligned_impl(s, av);
+  void* p = allocate_aligned_impl(s, av);
   if (p == nullptr)
     detail::throw_bad_alloc_helper();
   return p;
@@ -527,7 +527,7 @@ void* operator new[](std::size_t s, std::align_val_t av, std::nothrow_t const&)
     return nullptr;
   }
 #    endif
-  return alocate_aligned_impl(s, av);
+  return allocate_aligned_impl(s, av);
 }
 
 void operator delete[](void* p, std::align_val_t av) TEST_NOEXCEPT {
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index cf34a814fed4154..a7bb9bd47299e25 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -682,6 +682,8 @@ static int64_t getTlsTpOffset(const Symbol &s) {
   // before TP. The alignment padding is added so that (TP - padding -
   // p_memsz) is congruent to p_vaddr modulo p_align.
   PhdrEntry *tls = ctx.tlsPhdr;
+  if (!tls) // Reported an error in getSymVA
+    return 0;
   switch (config->emachine) {
     // Variant 1.
   case EM_ARM:
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index 4e1ae819b86fad3..13fc6dc0dd572a2 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -119,9 +119,11 @@ static uint64_t getSymVA(const Symbol &sym, int64_t addend) {
       // after sections are finalized. (e.g. Measuring the size of .rela.dyn
       // for Android relocation packing requires knowing TLS symbol addresses
       // during section finalization.)
-      if (!ctx.tlsPhdr || !ctx.tlsPhdr->firstSec)
-        fatal(toString(d.file) +
-              " has an STT_TLS symbol but doesn't have an SHF_TLS section");
+      if (!ctx.tlsPhdr || !ctx.tlsPhdr->firstSec) {
+        errorOrWarn(toString(d.file) +
+                    " has an STT_TLS symbol but doesn't have a PT_TLS segment");
+        return 0;
+      }
       return va - ctx.tlsPhdr->firstSec->addr;
     }
     return va;
diff --git a/lld/test/ELF/invalid/tls-symbol.s b/lld/test/ELF/invalid/tls-symbol.s
index 3f371d406e1f2f6..1af1e783d1f5b4e 100644
--- a/lld/test/ELF/invalid/tls-symbol.s
+++ b/lld/test/ELF/invalid/tls-symbol.s
@@ -4,7 +4,7 @@
 # RUN: yaml2obj %s -o %t.o
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
-# CHECK: has an STT_TLS symbol but doesn't have an SHF_TLS section
+# CHECK: has an STT_TLS symbol but doesn't have a PT_TLS segment
 
 --- !ELF
 FileHeader:      
diff --git a/lld/test/ELF/linkerscript/phdrs-no-tls.test b/lld/test/ELF/linkerscript/phdrs-no-tls.test
new file mode 100644
index 000000000000000..0d374beaf8eee86
--- /dev/null
+++ b/lld/test/ELF/linkerscript/phdrs-no-tls.test
@@ -0,0 +1,32 @@
+## Test STT_TLS and relocations without PT_TLS. See also invalid/tls-symbol.s.
+# REQUIRES: x86
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: not ld.lld -T a.lds a.o
+# RUN: ld.lld -T a.lds a.o --noinhibit-exec 2>&1 | FileCheck %s --implicit-check-not=warning:
+
+# CHECK:      warning: a.o has an STT_TLS symbol but doesn't have a PT_TLS segment
+# CHECK-NEXT: warning: a.o has an STT_TLS symbol but doesn't have a PT_TLS segment
+
+#--- a.lds
+PHDRS {
+  text PT_LOAD FLAGS(5);
+  data PT_LOAD FLAGS(6);
+}
+
+SECTIONS {
+  . = SIZEOF_HEADERS;
+  .text : { *(.text .text.*) } :text
+  .data : { *(.data .data.*) } :data
+}
+
+#--- a.s
+.globl _start
+_start:
+  movl %fs:a@TPOFF, %eax
+  movl %fs:b@TPOFF, %eax
+
+.section .tbss,"awT"
+a:
+b:
+  .long 0
diff --git a/lldb/include/lldb/Symbol/ObjectFile.h b/lldb/include/lldb/Symbol/ObjectFile.h
index 8592323322e383e..d89314d44bf6719 100644
--- a/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/lldb/include/lldb/Symbol/ObjectFile.h
@@ -656,8 +656,9 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
   // When an object file is in memory, subclasses should try and lock the
   // process weak pointer. If the process weak pointer produces a valid
   // ProcessSP, then subclasses can call this function to read memory.
-  static lldb::DataBufferSP ReadMemory(const lldb::ProcessSP &process_sp,
-                                       lldb::addr_t addr, size_t byte_size);
+  static lldb::WritableDataBufferSP
+  ReadMemory(const lldb::ProcessSP &process_sp, lldb::addr_t addr,
+             size_t byte_size);
 
   // This function returns raw file contents. Do not use it if you want
   // transparent decompression of section contents.
diff --git a/lldb/source/Expression/IRInterpreter.cpp b/lldb/source/Expression/IRInterpreter.cpp
index 5b670067b5c4337..030b30f070ee352 100644
--- a/lldb/source/Expression/IRInterpreter.cpp
+++ b/lldb/source/Expression/IRInterpreter.cpp
@@ -96,7 +96,7 @@ class InterpreterStackFrame {
   typedef std::map<const Value *, lldb::addr_t> ValueMap;
 
   ValueMap m_values;
-  DataLayout &m_target_data;
+  const DataLayout &m_target_data;
   lldb_private::IRExecutionUnit &m_execution_unit;
   const BasicBlock *m_bb = nullptr;
   const BasicBlock *m_prev_bb = nullptr;
@@ -110,7 +110,7 @@ class InterpreterStackFrame {
   lldb::ByteOrder m_byte_order;
   size_t m_addr_byte_size;
 
-  InterpreterStackFrame(DataLayout &target_data,
+  InterpreterStackFrame(const DataLayout &target_data,
                         lldb_private::IRExecutionUnit &execution_unit,
                         lldb::addr_t stack_frame_bottom,
                         lldb::addr_t stack_frame_top)
@@ -703,7 +703,7 @@ bool IRInterpreter::Interpret(llvm::Module &module, llvm::Function &function,
               s.c_str());
   }
 
-  DataLayout data_layout(&module);
+  const DataLayout &data_layout = module.getDataLayout();
 
   InterpreterStackFrame frame(data_layout, execution_unit, stack_frame_bottom,
                               stack_frame_top);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp
index bc0f5993aad0d6a..defd72bbd931062 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRDynamicChecks.cpp
@@ -280,10 +280,9 @@ class Instrumenter {
 
   IntegerType *GetIntptrTy() {
     if (!m_intptr_ty) {
-      llvm::DataLayout data_layout(&m_module);
-
-      m_intptr_ty = llvm::Type::getIntNTy(m_module.getContext(),
-                                          data_layout.getPointerSizeInBits());
+      m_intptr_ty = llvm::Type::getIntNTy(
+          m_module.getContext(),
+          m_module.getDataLayout().getPointerSizeInBits());
     }
 
     return m_intptr_ty;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
index cc9bd14c6194e4f..34461da46dfc7be 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.cpp
@@ -1606,7 +1606,7 @@ bool IRForTarget::runOnModule(Module &llvm_module) {
   lldb_private::Log *log(GetLog(LLDBLog::Expressions));
 
   m_module = &llvm_module;
-  m_target_data = std::make_unique<DataLayout>(m_module);
+  m_target_data = &m_module->getDataLayout();
   m_intptr_ty = llvm::Type::getIntNTy(m_module->getContext(),
                                       m_target_data->getPointerSizeInBits());
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.h b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.h
index a924187ba04c061..45027fcd6fa4921 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/IRForTarget.h
@@ -331,9 +331,9 @@ class IRForTarget {
   lldb_private::TypeFromParser m_result_type;
   /// The module being processed, or NULL if that has not been determined yet.
   llvm::Module *m_module = nullptr;
-  /// The target data for the module being processed, or NULL if there is no
+  /// The target data for the module being processed, or nullptr if there is no
   /// module.
-  std::unique_ptr<llvm::DataLayout> m_target_data;
+  const llvm::DataLayout *m_target_data = nullptr;
   /// The DeclMap containing the Decls
   lldb_private::ClangExpressionDeclMap *m_decl_map;
   /// The address of the function CFStringCreateWithBytes, cast to the
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 890db5c27481462..f348b94baad806c 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -419,19 +419,35 @@ ObjectFile *ObjectFileELF::CreateInstance(const lldb::ModuleSP &module_sp,
 ObjectFile *ObjectFileELF::CreateMemoryInstance(
     const lldb::ModuleSP &module_sp, WritableDataBufferSP data_sp,
     const lldb::ProcessSP &process_sp, lldb::addr_t header_addr) {
-  if (data_sp && data_sp->GetByteSize() > (llvm::ELF::EI_NIDENT)) {
-    const uint8_t *magic = data_sp->GetBytes();
-    if (ELFHeader::MagicBytesMatch(magic)) {
-      unsigned address_size = ELFHeader::AddressSizeInBytes(magic);
-      if (address_size == 4 || address_size == 8) {
-        std::unique_ptr<ObjectFileELF> objfile_up(
-            new ObjectFileELF(module_sp, data_sp, process_sp, header_addr));
-        ArchSpec spec = objfile_up->GetArchitecture();
-        if (spec && objfile_up->SetModulesArchitecture(spec))
-          return objfile_up.release();
-      }
-    }
-  }
+  if (!data_sp || data_sp->GetByteSize() < (llvm::ELF::EI_NIDENT))
+    return nullptr;
+  const uint8_t *magic = data_sp->GetBytes();
+  if (!ELFHeader::MagicBytesMatch(magic))
+    return nullptr;
+  // Read the ELF header first so we can figure out how many bytes we need
+  // to read to get as least the ELF header + program headers.
+  DataExtractor data;
+  data.SetData(data_sp);
+  elf::ELFHeader hdr;
+  lldb::offset_t offset = 0;
+  if (!hdr.Parse(data, &offset))
+    return nullptr;
+
+  // Make sure the address size is set correctly in the ELF header.
+  if (!hdr.Is32Bit() && !hdr.Is64Bit())
+    return nullptr;
+  // Figure out where the program headers end and read enough bytes to get the
+  // program headers in their entirety.
+  lldb::offset_t end_phdrs = hdr.e_phoff + (hdr.e_phentsize * hdr.e_phnum);
+  if (end_phdrs > data_sp->GetByteSize())
+    data_sp = ReadMemory(process_sp, header_addr, end_phdrs);
+
+  std::unique_ptr<ObjectFileELF> objfile_up(
+      new ObjectFileELF(module_sp, data_sp, process_sp, header_addr));
+  ArchSpec spec = objfile_up->GetArchitecture();
+  if (spec && objfile_up->SetModulesArchitecture(spec))
+    return objfile_up.release();
+
   return nullptr;
 }
 
@@ -873,42 +889,37 @@ Address ObjectFileELF::GetImageInfoAddress(Target *target) {
   if (!section_list)
     return Address();
 
-  // Find the SHT_DYNAMIC (.dynamic) section.
-  SectionSP dynsym_section_sp(
-      section_list->FindSectionByType(eSectionTypeELFDynamicLinkInfo, true));
-  if (!dynsym_section_sp)
-    return Address();
-  assert(dynsym_section_sp->GetObjectFile() == this);
+  for (size_t i = 0; i < m_dynamic_symbols.size(); ++i) {
+    const ELFDynamic &symbol = m_dynamic_symbols[i].symbol;
 
-  user_id_t dynsym_id = dynsym_section_sp->GetID();
-  const ELFSectionHeaderInfo *dynsym_hdr = GetSectionHeaderByIndex(dynsym_id);
-  if (!dynsym_hdr)
-    return Address();
+    if (symbol.d_tag != DT_DEBUG && symbol.d_tag != DT_MIPS_RLD_MAP &&
+        symbol.d_tag != DT_MIPS_RLD_MAP_REL)
+      continue;
 
-  for (size_t i = 0; i < m_dynamic_symbols.size(); ++i) {
-    ELFDynamic &symbol = m_dynamic_symbols[i];
+    // Compute the offset as the number of previous entries plus the size of
+    // d_tag.
+    const addr_t offset = (i * 2 + 1) * GetAddressByteSize();
+    const addr_t d_file_addr = m_dynamic_base_addr + offset;
+    Address d_addr;
+    if (!d_addr.ResolveAddressUsingFileSections(d_file_addr, GetSectionList()))
+      return Address();
+    if (symbol.d_tag == DT_DEBUG)
+      return d_addr;
 
-    if (symbol.d_tag == DT_DEBUG) {
-      // Compute the offset as the number of previous entries plus the size of
-      // d_tag.
-      addr_t offset = i * dynsym_hdr->sh_entsize + GetAddressByteSize();
-      return Address(dynsym_section_sp, offset);
-    }
     // MIPS executables uses DT_MIPS_RLD_MAP_REL to support PIE. DT_MIPS_RLD_MAP
     // exists in non-PIE.
-    else if ((symbol.d_tag == DT_MIPS_RLD_MAP ||
-              symbol.d_tag == DT_MIPS_RLD_MAP_REL) &&
-             target) {
-      addr_t offset = i * dynsym_hdr->sh_entsize + GetAddressByteSize();
-      addr_t dyn_base = dynsym_section_sp->GetLoadBaseAddress(target);
-      if (dyn_base == LLDB_INVALID_ADDRESS)
+    if ((symbol.d_tag == DT_MIPS_RLD_MAP ||
+         symbol.d_tag == DT_MIPS_RLD_MAP_REL) &&
+        target) {
+      const addr_t d_load_addr = d_addr.GetLoadAddress(target);
+      if (d_load_addr == LLDB_INVALID_ADDRESS)
         return Address();
 
       Status error;
       if (symbol.d_tag == DT_MIPS_RLD_MAP) {
         // DT_MIPS_RLD_MAP tag stores an absolute address of the debug pointer.
         Address addr;
-        if (target->ReadPointerFromMemory(dyn_base + offset, error, addr, true))
+        if (target->ReadPointerFromMemory(d_load_addr, error, addr, true))
           return addr;
       }
       if (symbol.d_tag == DT_MIPS_RLD_MAP_REL) {
@@ -916,18 +927,17 @@ Address ObjectFileELF::GetImageInfoAddress(Target *target) {
         // relative to the address of the tag.
         uint64_t rel_offset;
         rel_offset = target->ReadUnsignedIntegerFromMemory(
-            dyn_base + offset, GetAddressByteSize(), UINT64_MAX, error, true);
+            d_load_addr, GetAddressByteSize(), UINT64_MAX, error, true);
         if (error.Success() && rel_offset != UINT64_MAX) {
           Address addr;
           addr_t debug_ptr_address =
-              dyn_base + (offset - GetAddressByteSize()) + rel_offset;
+              d_load_addr - GetAddressByteSize() + rel_offset;
           addr.SetOffset(debug_ptr_address);
           return addr;
         }
       }
     }
   }
-
   return Address();
 }
 
@@ -970,62 +980,23 @@ Address ObjectFileELF::GetBaseAddress() {
   return LLDB_INVALID_ADDRESS;
 }
 
-// ParseDependentModules
 size_t ObjectFileELF::ParseDependentModules() {
   if (m_filespec_up)
     return m_filespec_up->GetSize();
 
   m_filespec_up = std::make_unique<FileSpecList>();
 
-  if (!ParseSectionHeaders())
-    return 0;
-
-  SectionList *section_list = GetSectionList();
-  if (!section_list)
-    return 0;
-
-  // Find the SHT_DYNAMIC section.
-  Section *dynsym =
-      section_list->FindSectionByType(eSectionTypeELFDynamicLinkInfo, true)
-          .get();
-  if (!dynsym)
-    return 0;
-  assert(dynsym->GetObjectFile() == this);
-
-  const ELFSectionHeaderInfo *header = GetSectionHeaderByIndex(dynsym->GetID());
-  if (!header)
-    return 0;
-  // sh_link: section header index of string table used by entries in the
-  // section.
-  Section *dynstr = section_list->FindSectionByID(header->sh_link).get();
-  if (!dynstr)
-    return 0;
-
-  DataExtractor dynsym_data;
-  DataExtractor dynstr_data;
-  if (ReadSectionData(dynsym, dynsym_data) &&
-      ReadSectionData(dynstr, dynstr_data)) {
-    ELFDynamic symbol;
-    const lldb::offset_t section_size = dynsym_data.GetByteSize();
-    lldb::offset_t offset = 0;
-
-    // The only type of entries we are concerned with are tagged DT_NEEDED,
-    // yielding the name of a required library.
-    while (offset < section_size) {
-      if (!symbol.Parse(dynsym_data, &offset))
-        break;
-
-      if (symbol.d_tag != DT_NEEDED)
+  if (ParseDynamicSymbols()) {
+    for (const auto &entry : m_dynamic_symbols) {
+      if (entry.symbol.d_tag != DT_NEEDED)
         continue;
-
-      uint32_t str_index = static_cast<uint32_t>(symbol.d_val);
-      const char *lib_name = dynstr_data.PeekCStr(str_index);
-      FileSpec file_spec(lib_name);
-      FileSystem::Instance().Resolve(file_spec);
-      m_filespec_up->Append(file_spec);
+      if (!entry.name.empty()) {
+        FileSpec file_spec(entry.name);
+        FileSystem::Instance().Resolve(file_spec);
+        m_filespec_up->Append(file_spec);
+      }
     }
   }
-
   return m_filespec_up->GetSize();
 }
 
@@ -2472,48 +2443,47 @@ size_t ObjectFileELF::ParseDynamicSymbols() {
   if (m_dynamic_symbols.size())
     return m_dynamic_symbols.size();
 
-  SectionList *section_list = GetSectionList();
-  if (!section_list)
+  std::optional<DataExtractor> dynamic_data = GetDynamicData();
+  if (!dynamic_data)
     return 0;
 
-  // Find the SHT_DYNAMIC section.
-  Section *dynsym =
-      section_list->FindSectionByType(eSectionTypeELFDynamicLinkInfo, true)
-          .get();
-  if (!dynsym)
-    return 0;
-  assert(dynsym->GetObjectFile() == this);
-
-  ELFDynamic symbol;
-  DataExtractor dynsym_data;
-  if (ReadSectionData(dynsym, dynsym_data)) {
-    const lldb::offset_t section_size = dynsym_data.GetByteSize();
-    lldb::offset_t cursor = 0;
-
-    while (cursor < section_size) {
-      if (!symbol.Parse(dynsym_data, &cursor))
+  ELFDynamicWithName e;
+  lldb::offset_t cursor = 0;
+  while (e.symbol.Parse(*dynamic_data, &cursor)) {
+    m_dynamic_symbols.push_back(e);
+    if (e.symbol.d_tag == DT_NULL)
+      break;
+  }
+  if (std::optional<DataExtractor> dynstr_data = GetDynstrData()) {
+    for (ELFDynamicWithName &entry : m_dynamic_symbols) {
+      switch (entry.symbol.d_tag) {
+      case DT_NEEDED:
+      case DT_SONAME:
+      case DT_RPATH:
+      case DT_RUNPATH:
+      case DT_AUXILIARY:
+      case DT_FILTER: {
+        lldb::offset_t cursor = entry.symbol.d_val;
+        const char *name = dynstr_data->GetCStr(&cursor);
+        if (name)
+          entry.name = std::string(name);
         break;
-
-      m_dynamic_symbols.push_back(symbol);
+      }
+      default:
+        break;
+      }
     }
   }
-
   return m_dynamic_symbols.size();
 }
 
 const ELFDynamic *ObjectFileELF::FindDynamicSymbol(unsigned tag) {
   if (!ParseDynamicSymbols())
     return nullptr;
-
-  DynamicSymbolCollIter I = m_dynamic_symbols.begin();
-  DynamicSymbolCollIter E = m_dynamic_symbols.end();
-  for (; I != E; ++I) {
-    ELFDynamic *symbol = &*I;
-
-    if (symbol->d_tag == tag)
-      return symbol;
+  for (const auto &entry : m_dynamic_symbols) {
+    if (entry.symbol.d_tag == tag)
+      return &entry.symbol;
   }
-
   return nullptr;
 }
 
@@ -3230,7 +3200,10 @@ void ObjectFileELF::Dump(Stream *s) {
   ArchSpec header_arch = GetArchitecture();
 
   *s << ", file = '" << m_file
-     << "', arch = " << header_arch.GetArchitectureName() << "\n";
+     << "', arch = " << header_arch.GetArchitectureName();
+  if (m_memory_addr != LLDB_INVALID_ADDRESS)
+    s->Printf(", addr = %#16.16" PRIx64, m_memory_addr);
+  s->EOL();
 
   DumpELFHeader(s, m_header);
   s->EOL();
@@ -3248,6 +3221,12 @@ void ObjectFileELF::Dump(Stream *s) {
   s->EOL();
   DumpDependentModules(s);
   s->EOL();
+  DumpELFDynamic(s);
+  s->EOL();
+  Address image_info_addr = GetImageInfoAddress(nullptr);
+  if (image_info_addr.IsValid())
+    s->Printf("image_info_address = %#16.16" PRIx64 "\n",
+              image_info_addr.GetFileAddress());
 }
 
 // DumpELFHeader
@@ -3492,6 +3471,111 @@ void ObjectFileELF::DumpDependentModules(lldb_private::Stream *s) {
   }
 }
 
+std::string static getDynamicTagAsString(uint16_t Arch, uint64_t Type) {
+#define DYNAMIC_STRINGIFY_ENUM(tag, value)                                     \
+  case value:                                                                  \
+    return #tag;
+
+#define DYNAMIC_TAG(n, v)
+  switch (Arch) {
+  case llvm::ELF::EM_AARCH64:
+    switch (Type) {
+#define AARCH64_DYNAMIC_TAG(name, value) DYNAMIC_STRINGIFY_ENUM(name, value)
+#include "llvm/BinaryFormat/DynamicTags.def"
+#undef AARCH64_DYNAMIC_TAG
+    }
+    break;
+
+  case llvm::ELF::EM_HEXAGON:
+    switch (Type) {
+#define HEXAGON_DYNAMIC_TAG(name, value) DYNAMIC_STRINGIFY_ENUM(name, value)
+#include "llvm/BinaryFormat/DynamicTags.def"
+#undef HEXAGON_DYNAMIC_TAG
+    }
+    break;
+
+  case llvm::ELF::EM_MIPS:
+    switch (Type) {
+#define MIPS_DYNAMIC_TAG(name, value) DYNAMIC_STRINGIFY_ENUM(name, value)
+#include "llvm/BinaryFormat/DynamicTags.def"
+#undef MIPS_DYNAMIC_TAG
+    }
+    break;
+
+  case llvm::ELF::EM_PPC:
+    switch (Type) {
+#define PPC_DYNAMIC_TAG(name, value) DYNAMIC_STRINGIFY_ENUM(name, value)
+#include "llvm/BinaryFormat/DynamicTags.def"
+#undef PPC_DYNAMIC_TAG
+    }
+    break;
+
+  case llvm::ELF::EM_PPC64:
+    switch (Type) {
+#define PPC64_DYNAMIC_TAG(name, value) DYNAMIC_STRINGIFY_ENUM(name, value)
+#include "llvm/BinaryFormat/DynamicTags.def"
+#undef PPC64_DYNAMIC_TAG
+    }
+    break;
+
+  case llvm::ELF::EM_RISCV:
+    switch (Type) {
+#define RISCV_DYNAMIC_TAG(name, value) DYNAMIC_STRINGIFY_ENUM(name, value)
+#include "llvm/BinaryFormat/DynamicTags.def"
+#undef RISCV_DYNAMIC_TAG
+    }
+    break;
+  }
+#undef DYNAMIC_TAG
+  switch (Type) {
+// Now handle all dynamic tags except the architecture specific ones
+#define AARCH64_DYNAMIC_TAG(name, value)
+#define MIPS_DYNAMIC_TAG(name, value)
+#define HEXAGON_DYNAMIC_TAG(name, value)
+#define PPC_DYNAMIC_TAG(name, value)
+#define PPC64_DYNAMIC_TAG(name, value)
+#define RISCV_DYNAMIC_TAG(name, value)
+// Also ignore marker tags such as DT_HIOS (maps to DT_VERNEEDNUM), etc.
+#define DYNAMIC_TAG_MARKER(name, value)
+#define DYNAMIC_TAG(name, value)                                               \
+  case value:                                                                  \
+    return #name;
+#include "llvm/BinaryFormat/DynamicTags.def"
+#undef DYNAMIC_TAG
+#undef AARCH64_DYNAMIC_TAG
+#undef MIPS_DYNAMIC_TAG
+#undef HEXAGON_DYNAMIC_TAG
+#undef PPC_DYNAMIC_TAG
+#undef PPC64_DYNAMIC_TAG
+#undef RISCV_DYNAMIC_TAG
+#undef DYNAMIC_TAG_MARKER
+#undef DYNAMIC_STRINGIFY_ENUM
+  default:
+    return "<unknown:>0x" + llvm::utohexstr(Type, true);
+  }
+}
+
+void ObjectFileELF::DumpELFDynamic(lldb_private::Stream *s) {
+  ParseDynamicSymbols();
+  if (m_dynamic_symbols.empty())
+    return;
+
+  s->PutCString(".dynamic:\n");
+  s->PutCString("IDX  d_tag            d_val/d_ptr\n");
+  s->PutCString("==== ---------------- ------------------\n");
+  uint32_t idx = 0;
+  for (const auto &entry : m_dynamic_symbols) {
+    s->Printf("[%2u] ", idx++);
+    s->Printf(
+        "%-16s 0x%16.16" PRIx64,
+        getDynamicTagAsString(m_header.e_machine, entry.symbol.d_tag).c_str(),
+        entry.symbol.d_ptr);
+    if (!entry.name.empty())
+      s->Printf(" \"%s\"", entry.name.c_str());
+    s->EOL();
+  }
+}
+
 ArchSpec ObjectFileELF::GetArchitecture() {
   if (!ParseHeader())
     return ArchSpec();
@@ -3664,7 +3748,24 @@ llvm::ArrayRef<ELFProgramHeader> ObjectFileELF::ProgramHeaders() {
 }
 
 DataExtractor ObjectFileELF::GetSegmentData(const ELFProgramHeader &H) {
-  return DataExtractor(m_data, H.p_offset, H.p_filesz);
+  // Try and read the program header from our cached m_data which can come from
+  // the file on disk being mmap'ed or from the initial part of the ELF file we
+  // read from memory and cached.
+  DataExtractor data = DataExtractor(m_data, H.p_offset, H.p_filesz);
+  if (data.GetByteSize() == H.p_filesz)
+    return data;
+  if (IsInMemory()) {
+    // We have a ELF file in process memory, read the program header data from
+    // the process.
+    if (ProcessSP process_sp = m_process_wp.lock()) {
+      const lldb::offset_t base_file_addr = GetBaseAddress().GetFileAddress();
+      const addr_t load_bias = m_memory_addr - base_file_addr;
+      const addr_t data_addr = H.p_vaddr + load_bias;
+      if (DataBufferSP data_sp = ReadMemory(process_sp, data_addr, H.p_memsz))
+        return DataExtractor(data_sp, GetByteOrder(), GetAddressByteSize());
+    }
+  }
+  return DataExtractor();
 }
 
 bool ObjectFileELF::AnySegmentHasPhysicalAddress() {
@@ -3704,3 +3805,88 @@ ObjectFileELF::MapFileDataWritable(const FileSpec &file, uint64_t Size,
   return FileSystem::Instance().CreateWritableDataBuffer(file.GetPath(), Size,
                                                          Offset);
 }
+
+std::optional<DataExtractor> ObjectFileELF::GetDynstrData() {
+  if (SectionList *section_list = GetSectionList()) {
+    // Find the SHT_DYNAMIC section.
+    if (Section *dynamic =
+            section_list
+                ->FindSectionByType(eSectionTypeELFDynamicLinkInfo, true)
+                .get()) {
+      assert(dynamic->GetObjectFile() == this);
+      if (const ELFSectionHeaderInfo *header =
+              GetSectionHeaderByIndex(dynamic->GetID())) {
+        // sh_link: section header index of string table used by entries in
+        // the section.
+        if (Section *dynstr =
+                section_list->FindSectionByID(header->sh_link).get()) {
+          DataExtractor data;
+          if (ReadSectionData(dynstr, data))
+            return data;
+        }
+      }
+    }
+  }
+
+  // Every ELF file which represents an executable or shared library has
+  // mandatory .dynamic entries. Two of these values are DT_STRTAB and DT_STRSZ
+  // and represent the dynamic symbol tables's string table. These are needed
+  // by the dynamic loader and we can read them from a process' address space.
+  //
+  // When loading and ELF file from memory, only the program headers end up
+  // being mapped into memory, and we can find these values in the PT_DYNAMIC
+  // segment.
+  const ELFDynamic *strtab = FindDynamicSymbol(DT_STRTAB);
+  const ELFDynamic *strsz = FindDynamicSymbol(DT_STRSZ);
+  if (strtab == nullptr || strsz == nullptr)
+    return std::nullopt;
+
+  if (ProcessSP process_sp = m_process_wp.lock()) {
+    if (DataBufferSP data_sp =
+            ReadMemory(process_sp, strtab->d_ptr, strsz->d_val))
+      return DataExtractor(data_sp, GetByteOrder(), GetAddressByteSize());
+  } else {
+    // We have an ELF file with no section headers or we didn't find the
+    // .dynamic section. Try and find the .dynstr section.
+    Address addr;
+    if (addr.ResolveAddressUsingFileSections(strtab->d_ptr, GetSectionList())) {
+      DataExtractor data;
+      addr.GetSection()->GetSectionData(data);
+      return DataExtractor(data,
+                           strtab->d_ptr - addr.GetSection()->GetFileAddress(),
+                           strsz->d_val);
+    }
+  }
+  return std::nullopt;
+}
+
+std::optional<lldb_private::DataExtractor> ObjectFileELF::GetDynamicData() {
+  DataExtractor data;
+  // The PT_DYNAMIC program header describes where the .dynamic section is and
+  // doesn't require parsing section headers. The PT_DYNAMIC is required by
+  // executables and shared libraries so it will always be available.
+  for (const ELFProgramHeader &H : ProgramHeaders()) {
+    if (H.p_type == llvm::ELF::PT_DYNAMIC) {
+      data = GetSegmentData(H);
+      if (data.GetByteSize() > 0) {
+        m_dynamic_base_addr = H.p_vaddr;
+        return data;
+      }
+    }
+  }
+  // Fall back to using section headers.
+  if (SectionList *section_list = GetSectionList()) {
+    // Find the SHT_DYNAMIC section.
+    if (Section *dynamic =
+            section_list
+                ->FindSectionByType(eSectionTypeELFDynamicLinkInfo, true)
+                .get()) {
+      assert(dynamic->GetObjectFile() == this);
+      if (ReadSectionData(dynamic, data)) {
+        m_dynamic_base_addr = dynamic->GetFileAddress();
+        return data;
+      }
+    }
+  }
+  return std::nullopt;
+}
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
index 844e981b1d890a3..aba3a5bfcbf5b6c 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
@@ -183,7 +183,11 @@ class ObjectFileELF : public lldb_private::ObjectFile {
   typedef SectionHeaderColl::iterator SectionHeaderCollIter;
   typedef SectionHeaderColl::const_iterator SectionHeaderCollConstIter;
 
-  typedef std::vector<elf::ELFDynamic> DynamicSymbolColl;
+  struct ELFDynamicWithName {
+    elf::ELFDynamic symbol;
+    std::string name;
+  };
+  typedef std::vector<ELFDynamicWithName> DynamicSymbolColl;
   typedef DynamicSymbolColl::iterator DynamicSymbolCollIter;
   typedef DynamicSymbolColl::const_iterator DynamicSymbolCollConstIter;
 
@@ -213,6 +217,10 @@ class ObjectFileELF : public lldb_private::ObjectFile {
   /// Collection of section headers.
   SectionHeaderColl m_section_headers;
 
+  /// The file address of the .dynamic section. This can be found in the p_vaddr
+  /// of the PT_DYNAMIC program header.
+  lldb::addr_t m_dynamic_base_addr = LLDB_INVALID_ADDRESS;
+
   /// Collection of symbols from the dynamic table.
   DynamicSymbolColl m_dynamic_symbols;
 
@@ -384,6 +392,9 @@ class ObjectFileELF : public lldb_private::ObjectFile {
   /// ELF dependent module dump routine.
   void DumpDependentModules(lldb_private::Stream *s);
 
+  /// ELF dump the .dynamic section
+  void DumpELFDynamic(lldb_private::Stream *s);
+
   const elf::ELFDynamic *FindDynamicSymbol(unsigned tag);
 
   unsigned PLTRelocationType();
@@ -402,6 +413,28 @@ class ObjectFileELF : public lldb_private::ObjectFile {
   /// .gnu_debugdata section or \c nullptr if an error occured or if there's no
   /// section with that name.
   std::shared_ptr<ObjectFileELF> GetGnuDebugDataObjectFile();
+
+  /// Get the bytes that represent the .dynamic section.
+  ///
+  /// This function will fetch the data for the .dynamic section in an ELF file.
+  /// The PT_DYNAMIC program header will be used to extract the data and this
+  /// function will fall back to using the section headers if PT_DYNAMIC isn't
+  /// found.
+  ///
+  /// \return The bytes that represent the string table data or \c std::nullopt
+  ///         if an error occured.
+  std::optional<lldb_private::DataExtractor> GetDynamicData();
+
+  /// Get the bytes that represent the dynamic string table data.
+  ///
+  /// This function will fetch the data for the string table in an ELF file. If
+  /// the ELF file is loaded from a file on disk, it will use the section
+  /// headers to extract the data and fall back to using the DT_STRTAB and
+  /// DT_STRSZ .dynamic entries.
+  ///
+  /// \return The bytes that represent the string table data or \c std::nullopt
+  ///         if an error occured.
+  std::optional<lldb_private::DataExtractor> GetDynstrData();
 };
 
 #endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_ELF_OBJECTFILEELF_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index 0a52159d055bb4c..66a762bf9b68542 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -97,14 +97,12 @@ void DWARFUnit::ExtractUnitDIEIfNeeded() {
         *m_dwo_id, m_first_die.GetOffset()));
     return; // Can't fetch the compile unit from the dwo file.
   }
-
-  // Link the DWO unit to this object, if it hasn't been linked already (this
-  // can happen when we have an index, and the DWO unit is parsed first).
-  if (!dwo_cu->LinkToSkeletonUnit(*this)) {
-    SetDwoError(Status::createWithFormat(
-        "multiple compile units with Dwo ID {0:x16}", *m_dwo_id));
-    return;
-  }
+  // If the skeleton compile unit gets its unit DIE parsed first, then this
+  // will fill in the DWO file's back pointer to this skeleton compile unit.
+  // If the DWO files get parsed on their own first the skeleton back link
+  // can be done manually in DWARFUnit::GetSkeletonCompileUnit() which will
+  // do a reverse lookup and cache the result.
+  dwo_cu->SetSkeletonUnit(this);
 
   DWARFBaseDIE dwo_cu_die = dwo_cu->GetUnitDIEOnly();
   if (!dwo_cu_die.IsValid()) {
@@ -720,11 +718,13 @@ DWARFCompileUnit *DWARFUnit::GetSkeletonUnit() {
   return llvm::dyn_cast_or_null<DWARFCompileUnit>(m_skeleton_unit);
 }
 
-bool DWARFUnit::LinkToSkeletonUnit(DWARFUnit &skeleton_unit) {
-  if (m_skeleton_unit && m_skeleton_unit != &skeleton_unit)
-    return false;
-  m_skeleton_unit = &skeleton_unit;
-  return true;
+void DWARFUnit::SetSkeletonUnit(DWARFUnit *skeleton_unit) {
+  // If someone is re-setting the skeleton compile unit backlink, make sure
+  // it is setting it to a valid value when it wasn't valid, or if the
+  // value in m_skeleton_unit was valid, it should be the same value.
+  assert(skeleton_unit);
+  assert(m_skeleton_unit == nullptr || m_skeleton_unit == skeleton_unit);
+  m_skeleton_unit = skeleton_unit;
 }
 
 bool DWARFUnit::Supports_DW_AT_APPLE_objc_complete_type() {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
index 209104fe3a054e7..85c37971ced8e0f 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -170,7 +170,7 @@ class DWARFUnit : public UserID {
   /// both cases correctly and avoids crashes.
   DWARFCompileUnit *GetSkeletonUnit();
 
-  bool LinkToSkeletonUnit(DWARFUnit &skeleton_unit);
+  void SetSkeletonUnit(DWARFUnit *skeleton_unit);
 
   bool Supports_DW_AT_APPLE_objc_complete_type();
 
diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp
index 2608a9c5fb79a2f..35317d209de1f9a 100644
--- a/lldb/source/Symbol/ObjectFile.cpp
+++ b/lldb/source/Symbol/ObjectFile.cpp
@@ -454,9 +454,10 @@ AddressClass ObjectFile::GetAddressClass(addr_t file_addr) {
   return AddressClass::eUnknown;
 }
 
-DataBufferSP ObjectFile::ReadMemory(const ProcessSP &process_sp,
-                                    lldb::addr_t addr, size_t byte_size) {
-  DataBufferSP data_sp;
+WritableDataBufferSP ObjectFile::ReadMemory(const ProcessSP &process_sp,
+                                            lldb::addr_t addr,
+                                            size_t byte_size) {
+  WritableDataBufferSP data_sp;
   if (process_sp) {
     std::unique_ptr<DataBufferHeap> data_up(new DataBufferHeap(byte_size, 0));
     Status error;
diff --git a/lldb/test/Shell/ObjectFile/ELF/Inputs/memory-elf.cpp b/lldb/test/Shell/ObjectFile/ELF/Inputs/memory-elf.cpp
new file mode 100644
index 000000000000000..9cae6c99c9f7616
--- /dev/null
+++ b/lldb/test/Shell/ObjectFile/ELF/Inputs/memory-elf.cpp
@@ -0,0 +1,5 @@
+#include <stdio.h>
+int main() {
+  printf("Hello World\n"); // Use something from libc.so
+  return 0;
+}
diff --git a/lldb/test/Shell/ObjectFile/ELF/elf-dynamic-no-shdrs.yaml b/lldb/test/Shell/ObjectFile/ELF/elf-dynamic-no-shdrs.yaml
new file mode 100644
index 000000000000000..e7b2f8d971c90b6
--- /dev/null
+++ b/lldb/test/Shell/ObjectFile/ELF/elf-dynamic-no-shdrs.yaml
@@ -0,0 +1,93 @@
+## This test verifies that loading an ELF file that has no section headers can
+## find the contents on the .dynamic section and the strings associated with
+## the .dynamic seciton.
+## - Loading the .dynamic section from the PT_DYNAMIC
+## This test will make a simple executable that links against libc.so and we
+## verify that we can find the DT_NEEDED entry with the shared library found
+## in the .dynamic dump from "image dump objfile"
+
+# RUN: yaml2obj %s -o %t
+
+# RUN: %lldb -b \
+# RUN:   -o "target create -d '%t'" \
+# RUN:   -o "image dump objfile" \
+# RUN:   | FileCheck %s --dump-input=always
+# CHECK: (lldb) image dump objfile
+# CHECK: Dumping headers for 1 module(s).
+# CHECK: ObjectFileELF, file =
+# CHECK: ELF Header
+# Make sure there are no section headers
+# CHECK: e_shnum = 0x00000000
+
+# Make sure we find the program headers and see a PT_DYNAMIC entry.
+# CHECK: Program Headers
+# CHECK: IDX  p_type          p_offset p_vaddr  p_paddr  p_filesz p_memsz  p_flags                   p_align
+# CHECK: ==== --------------- -------- -------- -------- -------- -------- ------------------------- --------
+# CHECK: [ 0] PT_LOAD         000000b0 00000000 00000000 00000170 00000170 00000000 (              ) 00000001
+# CHECK: [ 1] PT_DYNAMIC      000001b0 00000100 00000100 00000070 00000070 00000000 (              ) 00000008
+
+# CHECK: Dependent Modules:
+# CHECK:    ccc
+# CHECK:    aaa
+# CHECK:    bbb
+
+# Make sure we see some sections created from the program headers
+# MAIN: SectID
+# MAIN: PT_LOAD[0]
+
+# CHECK: .dynamic:
+# CHECK: IDX  d_tag            d_val/d_ptr
+# CHECK: ==== ---------------- ------------------
+# CHECK: [ 0] STRTAB           0x0000000000000000
+# CHECK: [ 1] NEEDED           0x0000000000000009 "ccc"
+# CHECK: [ 2] NEEDED           0x0000000000000001 "aaa"
+# CHECK: [ 3] NEEDED           0x0000000000000005 "bbb"
+# CHECK: [ 4] STRSZ            0x0000000000000100
+# CHECK: [ 5] DEBUG            0x00000000deadbeef
+# CHECK: [ 6] NULL             0x0000000000000000
+
+# Make sure the ObjectFileELF::GetImageInfoAddress() works.
+# CHECK: image_info_address = 0x0000000000000158
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Type:      SectionHeaderTable
+    NoHeaders: true
+  - Name:  .dynstr
+    Type:  SHT_STRTAB
+    Flags: [ SHF_ALLOC ]
+    Content: '00616161006262620063636300' ## 0,a,a,a,0,b,b,b,0,c,c,c,0
+    Size:    0x100
+  - Name:    .dynamic
+    Type:    SHT_DYNAMIC
+    Address: 0x100
+    Entries:
+      - Tag:   DT_STRTAB
+        Value: 0x0000000000000000
+      - Tag:   DT_NEEDED
+        Value: 9
+      - Tag:   DT_NEEDED
+        Value: 1
+      - Tag:   DT_NEEDED
+        Value: 5
+      - Tag:   DT_STRSZ
+        Value: 0x100
+      - Tag:   DT_DEBUG
+        Value: 0xdeadbeef
+      - Tag:   DT_NULL
+        Value: 0x0
+ProgramHeaders:
+  - Type: PT_LOAD
+    VAddr: 0x0
+    FirstSec: .dynstr
+    LastSec:  .dynamic
+  - Type:            PT_DYNAMIC
+    FirstSec:        .dynamic
+    LastSec:         .dynamic
+    VAddr:           0x100
+    Align:           0x8
diff --git a/lldb/test/Shell/ObjectFile/ELF/elf-memory.test b/lldb/test/Shell/ObjectFile/ELF/elf-memory.test
new file mode 100644
index 000000000000000..0b1c01486a4b431
--- /dev/null
+++ b/lldb/test/Shell/ObjectFile/ELF/elf-memory.test
@@ -0,0 +1,55 @@
+// REQUIRES: system-linux, native
+
+// This test verifies that loading an ELF file from memory works and the new
+// features that were added when loading from memory work like:
+// - Loading the .dynamic section from the PT_DYNAMIC since ELF files loaded
+//   from memory don't have the section headers available.
+// This test will make a simple executable that:
+//   - links against libc
+//   - runs and stops at a breakpoint
+//   - create a memory ELF file
+//   - verify that "image dump objfile" will dump the dynamic section of the
+//     memory elf file and find the .dynamic string table.
+
+// RUN: %clang_host %p/Inputs/memory-elf.cpp -g -O0 -o %t
+
+// RUN: %lldb %t -b \
+// RUN:   -o "b main" \
+// RUN:   -o "run" \
+// RUN:   -o "script real_module = lldb.target.module[0]" \
+// RUN:   -o "script base_addr = real_module.GetObjectFileHeaderAddress().GetLoadAddress(lldb.target)" \
+// RUN:   -o "script mem_module = lldb.SBModule(lldb.process, base_addr)" \
+// RUN:   -o "script target2 = lldb.debugger.CreateTarget('')" \
+// RUN:   -o "script target2.AddModule(mem_module)" \
+// RUN:   -o "target select 1" \
+// RUN:   -o "image dump objfile" \
+// RUN:   | FileCheck %s --check-prefix=MAIN --dump-input=always
+// MAIN: (lldb) image dump objfile
+// MAIN: Dumping headers for 1 module(s).
+// MAIN: ObjectFileELF, file = '', arch = {{.*, addr = 0x[0-9a-f]+}}
+// MAIN: ELF Header
+
+// Make sure we find the program headers and see a PT_DYNAMIC entry.
+// MAIN: Program Headers
+// MAIN: ] PT_DYNAMIC
+
+// Make sure we see some sections created from the program headers
+// MAIN: SectID
+// MAIN: PT_LOAD[0]
+
+// Ensure we find some dependent modules as won't find these if we aren't able
+// to load the .dynamic section from the PT_DYNAMIC program header.
+// MAIN: Dependent Modules:
+
+// Check for the .dynamic dump and ensure we find all dynamic entries that are
+// required to be there and needed to get the .dynstr section and the symbol
+// table, and the DT_DEBUG entry to find the list of shared libraries.
+// MAIN: .dynamic:
+// Make sure we found the .dynstr section by checking for valid strings after NEEDED
+// MAIN-DAG: NEEDED {{0x[0-9a-f]+ ".*libc.*"}}
+// MAIN-DAG: STRTAB {{0x[0-9a-f]+}}
+// MAIN-DAG: SYMTAB {{0x[0-9a-f]+}}
+// MAIN-DAG: STRSZ {{0x[0-9a-f]+}}
+// MAIN-DAG: SYMENT {{0x[0-9a-f]+}}
+// MAIN-DAG: DEBUG {{0x[0-9a-f]+}}
+// MAIN:     NULL {{0x[0-9a-f]+}}
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s
deleted file mode 100644
index d626b4602ad58f4..000000000000000
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s
+++ /dev/null
@@ -1,142 +0,0 @@
-## Test that lldb handles (mainly, that it doesn't crash) the situation where
-## two skeleton compile units have the same DWO ID (and try to claim the same
-## split unit from the DWP file. This can sometimes happen when the compile unit
-## is nearly empty (e.g. because LTO has optimized all of it away).
-
-# RUN: llvm-mc -triple=x86_64-pc-linux -filetype=obj %s --defsym MAIN=0 > %t
-# RUN: llvm-mc -triple=x86_64-pc-linux -filetype=obj %s > %t.dwp
-# RUN: %lldb %t -o "image lookup -t my_enum_type" \
-# RUN:   -o "image dump separate-debug-info" -o exit | FileCheck %s
-
-## Check that we're able to access the type within the split unit (no matter
-## which skeleton unit it ends up associated with). Completely ignoring the unit
-## might also be reasonable.
-# CHECK: image lookup -t my_enum_type
-# CHECK: 1 match found
-# CHECK:      name = "my_enum_type", byte-size = 4, compiler_type = "enum my_enum_type {
-# CHECK-NEXT: }"
-#
-## Check that we get some indication of the error.
-# CHECK: image dump separate-debug-info
-# CHECK:      Dwo ID             Err Dwo Path
-# CHECK: 0xdeadbeefbaadf00d E   multiple compile units with Dwo ID 0xdeadbeefbaadf00d
-
-.set DWO_ID, 0xdeadbeefbaadf00d
-
-## The main file.
-.ifdef MAIN
-        .section        .debug_abbrev,"",@progbits
-        .byte   1                       # Abbreviation Code
-        .byte   74                      # DW_TAG_compile_unit
-        .byte   0                       # DW_CHILDREN_no
-        .byte   0x76                    # DW_AT_dwo_name
-        .byte   8                       # DW_FORM_string
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   0                       # EOM(3)
-
-
-        .section        .debug_info,"",@progbits
-.irpc I,01
-.Lcu_begin\I:
-        .long   .Ldebug_info_end\I-.Ldebug_info_start\I # Length of Unit
-.Ldebug_info_start\I:
-        .short  5                       # DWARF version number
-        .byte   4                       # DWARF Unit Type
-        .byte   8                       # Address Size (in bytes)
-        .long   .debug_abbrev           # Offset Into Abbrev. Section
-        .quad   DWO_ID                  # DWO id
-        .byte   1                       # Abbrev [1] DW_TAG_compile_unit
-        .ascii  "foo"
-        .byte   '0' + \I
-        .asciz  ".dwo\0"                # DW_AT_dwo_name
-.Ldebug_info_end\I:
-.endr
-
-.else
-## DWP file starts here.
-
-        .section        .debug_abbrev.dwo,"e",@progbits
-.LAbbrevBegin:
-        .byte   1                       # Abbreviation Code
-        .byte   17                      # DW_TAG_compile_unit
-        .byte   1                       # DW_CHILDREN_yes
-        .byte   37                      # DW_AT_producer
-        .byte   8                       # DW_FORM_string
-        .byte   19                      # DW_AT_language
-        .byte   5                       # DW_FORM_data2
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   2                       # Abbreviation Code
-        .byte   4                       # DW_TAG_enumeration_type
-        .byte   0                       # DW_CHILDREN_no
-        .byte   3                       # DW_AT_name
-        .byte   8                       # DW_FORM_string
-        .byte   73                      # DW_AT_type
-        .byte   19                      # DW_FORM_ref4
-        .byte   11                      # DW_AT_byte_size
-        .byte   11                      # DW_FORM_data1
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   4                       # Abbreviation Code
-        .byte   36                      # DW_TAG_base_type
-        .byte   0                       # DW_CHILDREN_no
-        .byte   3                       # DW_AT_name
-        .byte   8                       # DW_FORM_string
-        .byte   62                      # DW_AT_encoding
-        .byte   11                      # DW_FORM_data1
-        .byte   11                      # DW_AT_byte_size
-        .byte   11                      # DW_FORM_data1
-        .byte   0                       # EOM(1)
-        .byte   0                       # EOM(2)
-        .byte   0                       # EOM(3)
-.LAbbrevEnd:
-        .section        .debug_info.dwo,"e",@progbits
-.LCUBegin:
-.Lcu_begin1:
-        .long   .Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit
-.Ldebug_info_start1:
-        .short  5                       # DWARF version number
-        .byte   5                       # DWARF Unit Type
-        .byte   8                       # Address Size (in bytes)
-        .long   0                       # Offset Into Abbrev. Section
-        .quad   DWO_ID                  # DWO id
-        .byte   1                       # Abbrev [1] DW_TAG_compile_unit
-        .asciz  "Hand-written DWARF"    # DW_AT_producer
-        .short  12                      # DW_AT_language
-        .byte   2                       # Abbrev [2] DW_TAG_enumeration_type
-        .asciz  "my_enum_type"          # DW_AT_name
-        .long   .Lint-.Lcu_begin1       # DW_AT_type
-        .byte   4                       # DW_AT_byte_size
-.Lint:
-        .byte   4                       # Abbrev [4] DW_TAG_base_type
-        .asciz  "int"                   # DW_AT_name
-        .byte   5                       # DW_AT_encoding
-        .byte   4                       # DW_AT_byte_size
-        .byte   0                       # End Of Children Mark
-.Ldebug_info_end1:
-.LCUEnd:
-        .section .debug_cu_index, "", @progbits
-## Header:
-        .short 5                        # Version
-        .short 0                        # Padding
-        .long 2                         # Section count
-        .long 1                         # Unit count
-        .long 2                         # Slot count
-## Hash Table of Signatures:
-        .quad 0
-        .quad DWO_ID
-## Parallel Table of Indexes:
-        .long 0
-        .long 1
-## Table of Section Offsets:
-## Row 0:
-        .long 1                         # DW_SECT_INFO
-        .long 3                         # DW_SECT_ABBREV
-## Row 1:
-        .long 0                         # Offset in .debug_info.dwo
-        .long 0                         # Offset in .debug_abbrev.dwo
-## Table of Section Sizes:
-        .long .LCUEnd-.LCUBegin         # Size in .debug_info.dwo
-        .long .LAbbrevEnd-.LAbbrevBegin # Size in .debug_abbrev.dwo
-.endif
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index 7148a1d2a30941f..82a3a0d6b4e51c0 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -22,6 +22,7 @@
 #include <optional>
 
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -32,8 +33,10 @@
 #include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/HostGetOpt.h"
 #include "lldb/Host/OptionParser.h"
+#include "lldb/Host/Socket.h"
 #include "lldb/Host/common/TCPSocket.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Status.h"
 
 using namespace lldb;
@@ -44,6 +47,108 @@ using namespace llvm;
 
 // option descriptors for getopt_long_only()
 
+#ifdef _WIN32
+typedef pipe_t shared_fd_t;
+const shared_fd_t kInvalidSharedFD = LLDB_INVALID_PIPE;
+#else
+typedef NativeSocket shared_fd_t;
+const shared_fd_t kInvalidSharedFD = Socket::kInvalidSocketValue;
+#endif
+
+class SharedSocket {
+public:
+  SharedSocket(Connection *conn, Status &error) {
+    m_fd = kInvalidSharedFD;
+
+    const Socket *socket =
+        static_cast<const Socket *>(conn->GetReadObject().get());
+    if (socket == nullptr) {
+      error = Status("invalid conn socket");
+      return;
+    }
+
+#ifdef _WIN32
+    m_socket = socket->GetNativeSocket();
+
+    // Create a pipe to transfer WSAPROTOCOL_INFO to the child process.
+    error = m_socket_pipe.CreateNew(true);
+    if (error.Fail())
+      return;
+
+    m_fd = m_socket_pipe.GetReadPipe();
+#else
+    m_fd = socket->GetNativeSocket();
+    error = Status();
+#endif
+  }
+
+  shared_fd_t GetSendableFD() { return m_fd; }
+
+  Status CompleteSending(lldb::pid_t child_pid) {
+#ifdef _WIN32
+    // Transfer WSAPROTOCOL_INFO to the child process.
+    m_socket_pipe.CloseReadFileDescriptor();
+
+    WSAPROTOCOL_INFO protocol_info;
+    if (::WSADuplicateSocket(m_socket, child_pid, &protocol_info) ==
+        SOCKET_ERROR) {
+      int last_error = ::WSAGetLastError();
+      return Status("WSADuplicateSocket() failed, error: %d", last_error);
+    }
+
+    size_t num_bytes;
+    Status error =
+        m_socket_pipe.WriteWithTimeout(&protocol_info, sizeof(protocol_info),
+                                       std::chrono::seconds(10), num_bytes);
+    if (error.Fail())
+      return error;
+    if (num_bytes != sizeof(protocol_info))
+      return Status("WriteWithTimeout(WSAPROTOCOL_INFO) failed: %d bytes",
+                    num_bytes);
+#endif
+    return Status();
+  }
+
+  static Status GetNativeSocket(shared_fd_t fd, NativeSocket &socket) {
+#ifdef _WIN32
+    socket = Socket::kInvalidSocketValue;
+    // Read WSAPROTOCOL_INFO from the parent process and create NativeSocket.
+    WSAPROTOCOL_INFO protocol_info;
+    {
+      Pipe socket_pipe(fd, LLDB_INVALID_PIPE);
+      size_t num_bytes;
+      Status error =
+          socket_pipe.ReadWithTimeout(&protocol_info, sizeof(protocol_info),
+                                      std::chrono::seconds(10), num_bytes);
+      if (error.Fail())
+        return error;
+      if (num_bytes != sizeof(protocol_info)) {
+        return Status(
+            "socket_pipe.ReadWithTimeout(WSAPROTOCOL_INFO) failed: % d bytes",
+            num_bytes);
+      }
+    }
+    socket = ::WSASocket(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO,
+                         FROM_PROTOCOL_INFO, &protocol_info, 0, 0);
+    if (socket == INVALID_SOCKET) {
+      return Status("WSASocket(FROM_PROTOCOL_INFO) failed: error %d",
+                    ::WSAGetLastError());
+    }
+    return Status();
+#else
+    socket = fd;
+    return Status();
+#endif
+  }
+
+private:
+#ifdef _WIN32
+  Pipe m_socket_pipe;
+  NativeSocket m_socket;
+#endif
+  shared_fd_t m_fd;
+};
+
 static int g_debug = 0;
 static int g_verbose = 0;
 static int g_server = 0;
@@ -60,6 +165,7 @@ static struct option g_long_options[] = {
     {"max-gdbserver-port", required_argument, nullptr, 'M'},
     {"socket-file", required_argument, nullptr, 'f'},
     {"server", no_argument, &g_server, 1},
+    {"child-platform-fd", required_argument, nullptr, 2},
     {nullptr, 0, nullptr, 0}};
 
 #if defined(__APPLE__)
@@ -114,6 +220,130 @@ static Status save_socket_id_to_file(const std::string &socket_id,
   return status;
 }
 
+static void client_handle(GDBRemoteCommunicationServerPlatform &platform,
+                          const lldb_private::Args &args) {
+  if (!platform.IsConnected())
+    return;
+
+  if (args.GetArgumentCount() > 0) {
+    lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
+    std::optional<uint16_t> port;
+    std::string socket_name;
+    Status error = platform.LaunchGDBServer(args,
+                                            "", // hostname
+                                            pid, port, socket_name);
+    if (error.Success())
+      platform.SetPendingGdbServer(pid, *port, socket_name);
+    else
+      fprintf(stderr, "failed to start gdbserver: %s\n", error.AsCString());
+  }
+
+  bool interrupt = false;
+  bool done = false;
+  Status error;
+  while (!interrupt && !done) {
+    if (platform.GetPacketAndSendResponse(std::nullopt, error, interrupt,
+                                          done) !=
+        GDBRemoteCommunication::PacketResult::Success)
+      break;
+  }
+
+  printf("Disconnected.\n");
+}
+
+static GDBRemoteCommunicationServerPlatform::PortMap gdbserver_portmap;
+static std::mutex gdbserver_portmap_mutex;
+
+static void spawn_process_reaped(lldb::pid_t pid, int signal, int status) {
+  std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
+  gdbserver_portmap.FreePortForProcess(pid);
+}
+
+static Status spawn_process(const char *progname, Connection *conn,
+                            uint16_t gdb_port, uint16_t port_offset,
+                            const lldb_private::Args &args,
+                            const std::string &log_file,
+                            const StringRef log_channels) {
+  Status error;
+  SharedSocket shared_socket(conn, error);
+  if (error.Fail())
+    return error;
+
+  ProcessLaunchInfo launch_info;
+
+  FileSpec self_spec(progname, FileSpec::Style::native);
+  launch_info.SetExecutableFile(self_spec, true);
+  Args &self_args = launch_info.GetArguments();
+  self_args.AppendArgument(llvm::StringRef("platform"));
+  self_args.AppendArgument(llvm::StringRef("--child-platform-fd"));
+  self_args.AppendArgument(llvm::to_string(shared_socket.GetSendableFD()));
+  if (gdb_port) {
+    self_args.AppendArgument(llvm::StringRef("--gdbserver-port"));
+    self_args.AppendArgument(llvm::to_string(gdb_port));
+  }
+  if (port_offset > 0) {
+    self_args.AppendArgument(llvm::StringRef("--port-offset"));
+    self_args.AppendArgument(llvm::to_string(port_offset));
+  }
+  if (!log_file.empty()) {
+    self_args.AppendArgument(llvm::StringRef("--log-file"));
+    self_args.AppendArgument(log_file);
+  }
+  if (!log_channels.empty()) {
+    self_args.AppendArgument(llvm::StringRef("--log-channels"));
+    self_args.AppendArgument(log_channels);
+  }
+  if (args.GetArgumentCount() > 0) {
+    self_args.AppendArgument("--");
+    self_args.AppendArguments(args);
+  }
+
+  launch_info.SetLaunchInSeparateProcessGroup(false);
+  launch_info.SetMonitorProcessCallback(&spawn_process_reaped);
+
+  // Copy the current environment.
+  launch_info.GetEnvironment() = Host::GetEnvironment();
+
+  launch_info.GetFlags().Set(eLaunchFlagDisableSTDIO);
+
+  // Close STDIN, STDOUT and STDERR.
+  launch_info.AppendCloseFileAction(STDIN_FILENO);
+  launch_info.AppendCloseFileAction(STDOUT_FILENO);
+  launch_info.AppendCloseFileAction(STDERR_FILENO);
+
+  // Redirect STDIN, STDOUT and STDERR to "/dev/null".
+  launch_info.AppendSuppressFileAction(STDIN_FILENO, true, false);
+  launch_info.AppendSuppressFileAction(STDOUT_FILENO, false, true);
+  launch_info.AppendSuppressFileAction(STDERR_FILENO, false, true);
+
+  std::string cmd;
+  self_args.GetCommandString(cmd);
+
+  error = Host::LaunchProcess(launch_info);
+  if (error.Fail())
+    return error;
+
+  lldb::pid_t child_pid = launch_info.GetProcessID();
+  if (child_pid == LLDB_INVALID_PROCESS_ID)
+    return Status("invalid pid");
+
+  LLDB_LOG(GetLog(LLDBLog::Platform), "lldb-platform launched '{0}', pid={1}",
+           cmd, child_pid);
+
+  {
+    std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
+    gdbserver_portmap.AssociatePortWithProcess(gdb_port, child_pid);
+  }
+
+  error = shared_socket.CompleteSending(child_pid);
+  if (error.Fail()) {
+    Host::Kill(child_pid, SIGTERM);
+    return error;
+  }
+
+  return Status();
+}
+
 // main
 int main_platform(int argc, char *argv[]) {
   const char *progname = argv[0];
@@ -133,7 +363,8 @@ int main_platform(int argc, char *argv[]) {
   StringRef
       log_channels; // e.g. "lldb process threads:gdb-remote default:linux all"
 
-  GDBRemoteCommunicationServerPlatform::PortMap gdbserver_portmap;
+  shared_fd_t fd = kInvalidSharedFD;
+
   int min_gdbserver_port = 0;
   int max_gdbserver_port = 0;
   uint16_t port_offset = 0;
@@ -217,6 +448,15 @@ int main_platform(int argc, char *argv[]) {
         max_gdbserver_port = portnum;
     } break;
 
+    case 2: {
+      uint64_t _fd;
+      if (!llvm::to_integer(optarg, _fd)) {
+        WithColor::error() << "invalid fd " << optarg << "\n";
+        option_error = 6;
+      } else
+        fd = (shared_fd_t)_fd;
+    } break;
+
     case 'h': /* fall-through is intentional */
     case '?':
       show_usage = true;
@@ -240,7 +480,7 @@ int main_platform(int argc, char *argv[]) {
   }
 
   // Print usage and exit if no listening port is specified.
-  if (listen_host_port.empty())
+  if (listen_host_port.empty() && fd == kInvalidSharedFD)
     show_usage = true;
 
   if (show_usage || option_error) {
@@ -254,6 +494,33 @@ int main_platform(int argc, char *argv[]) {
   lldb_private::Args inferior_arguments;
   inferior_arguments.SetArguments(argc, const_cast<const char **>(argv));
 
+  if (fd != kInvalidSharedFD) {
+    // Child process will handle the connection and exit.
+    Log *log = GetLog(LLDBLog::Platform);
+    if (!listen_host_port.empty()) {
+      LLDB_LOGF(log, "lldb-platform child: "
+                     "ambiguous parameters --listen and --child-platform-fd");
+      return socket_error;
+    }
+
+    NativeSocket socket;
+    error = SharedSocket::GetNativeSocket(fd, socket);
+    if (error.Fail()) {
+      LLDB_LOGF(log, "lldb-platform child: %s", error.AsCString());
+      return socket_error;
+    }
+
+    Connection *conn =
+        new ConnectionFileDescriptor(new TCPSocket(socket, true, false));
+    GDBRemoteCommunicationServerPlatform platform(Socket::ProtocolTcp, "tcp");
+    if (port_offset > 0)
+      platform.SetPortOffset(port_offset);
+    platform.SetPortMap(std::move(gdbserver_portmap));
+    platform.SetConnection(std::unique_ptr<Connection>(conn));
+    client_handle(platform, inferior_arguments);
+    return 0;
+  }
+
   const bool children_inherit_listen_socket = false;
   // the test suite makes many connections in parallel, let's not miss any.
   // The highest this should get reasonably is a function of the number
@@ -298,48 +565,38 @@ int main_platform(int argc, char *argv[]) {
     printf("Connection established.\n");
 
     if (g_server) {
-      // Collect child zombie processes.
-#if !defined(_WIN32)
-      ::pid_t waitResult;
-      while ((waitResult = waitpid(-1, nullptr, WNOHANG)) > 0) {
-        // waitResult is the child pid
-        gdbserver_portmap.FreePortForProcess(waitResult);
+      std::optional<uint16_t> available_port;
+      {
+        std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
+        auto port = gdbserver_portmap.GetNextAvailablePort();
+        if (port)
+          available_port = *port;
+        else
+          llvm::consumeError(port.takeError());
       }
-#endif
-      // TODO: Clean up portmap for Windows when children die
-      // See https://github.com/llvm/llvm-project/issues/90923
-
-      // After collecting zombie ports, get the next available
-      GDBRemoteCommunicationServerPlatform::PortMap portmap_for_child;
-      llvm::Expected<uint16_t> available_port =
-          gdbserver_portmap.GetNextAvailablePort();
-      if (available_port) {
-        // GetNextAvailablePort() may return 0 if gdbserver_portmap is empty.
-        if (*available_port)
-          portmap_for_child.AllowPort(*available_port);
-      } else {
-        llvm::consumeError(available_port.takeError());
+      if (!available_port) {
         fprintf(stderr,
                 "no available gdbserver port for connection - dropping...\n");
-        delete conn;
-        continue;
-      }
-      platform.SetPortMap(std::move(portmap_for_child));
-
-      auto childPid = fork();
-      if (childPid) {
-        gdbserver_portmap.AssociatePortWithProcess(*available_port, childPid);
-        // Parent doesn't need a connection to the lldb client
-        delete conn;
-
-        // Parent will continue to listen for new connections.
-        continue;
       } else {
-        // Child process will handle the connection and exit.
-        g_server = 0;
-        // Listening socket is owned by parent process.
-        acceptor_up.release();
+        error = spawn_process(progname, conn, *available_port, port_offset,
+                              inferior_arguments, log_file, log_channels);
+        if (error.Fail()) {
+          {
+
+            std::lock_guard<std::mutex> guard(gdbserver_portmap_mutex);
+            gdbserver_portmap.FreePort(*available_port);
+          }
+          LLDB_LOGF(GetLog(LLDBLog::Platform), "spawn_process failed: %s",
+                    error.AsCString());
+          WithColor::error()
+              << "spawn_process failed: " << error.AsCString() << "\n";
+        }
       }
+      // Parent doesn't need a connection to the lldb client
+      delete conn;
+
+      // Parent will continue to listen for new connections.
+      continue;
     } else {
       // If not running as a server, this process will not accept
       // connections while a connection is active.
@@ -350,33 +607,7 @@ int main_platform(int argc, char *argv[]) {
     }
 
     platform.SetConnection(std::unique_ptr<Connection>(conn));
-
-    if (platform.IsConnected()) {
-      if (inferior_arguments.GetArgumentCount() > 0) {
-        lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
-        std::optional<uint16_t> port;
-        std::string socket_name;
-        Status error = platform.LaunchGDBServer(inferior_arguments,
-                                                "", // hostname
-                                                pid, port, socket_name);
-        if (error.Success())
-          platform.SetPendingGdbServer(pid, *port, socket_name);
-        else
-          fprintf(stderr, "failed to start gdbserver: %s\n", error.AsCString());
-      }
-
-      bool interrupt = false;
-      bool done = false;
-      while (!interrupt && !done) {
-        if (platform.GetPacketAndSendResponse(std::nullopt, error, interrupt,
-                                              done) !=
-            GDBRemoteCommunication::PacketResult::Success)
-          break;
-      }
-
-      if (error.Fail())
-        WithColor::error() << error.AsCString() << '\n';
-    }
+    client_handle(platform, inferior_arguments);
   } while (g_server);
 
   fprintf(stderr, "lldb-server exiting...\n");
diff --git a/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h b/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
new file mode 100644
index 000000000000000..e5c6ae5db559eef
--- /dev/null
+++ b/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
@@ -0,0 +1,83 @@
+//=- DXILMetadataAnalysis.h - Representation of Module metadata --*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_DXILMETADATA_H
+#define LLVM_ANALYSIS_DXILMETADATA_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/VersionTuple.h"
+#include "llvm/TargetParser/Triple.h"
+#include <memory>
+
+namespace llvm {
+
+namespace dxil {
+
+struct ModuleMetadataInfo {
+  VersionTuple DXILVersion{};
+  VersionTuple ShaderModelVersion{};
+  Triple::EnvironmentType ShaderStage = Triple::UnknownEnvironment;
+
+  void print(raw_ostream &OS) const;
+};
+
+} // namespace dxil
+
+// Module metadata analysis pass for new pass manager
+class DXILMetadataAnalysis : public AnalysisInfoMixin<DXILMetadataAnalysis> {
+  friend AnalysisInfoMixin<DXILMetadataAnalysis>;
+
+  static AnalysisKey Key;
+
+public:
+  using Result = dxil::ModuleMetadataInfo;
+  /// Gather module metadata info for the module \c M.
+  dxil::ModuleMetadataInfo run(Module &M, ModuleAnalysisManager &AM);
+};
+
+/// Printer pass for the \c DXILMetadataAnalysis results.
+class DXILMetadataAnalysisPrinterPass
+    : public PassInfoMixin<DXILMetadataAnalysisPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  explicit DXILMetadataAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {}
+
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+
+  static bool isRequired() { return true; }
+};
+
+/// Legacy pass
+class DXILMetadataAnalysisWrapperPass : public ModulePass {
+  std::unique_ptr<dxil::ModuleMetadataInfo> MetadataInfo;
+
+public:
+  static char ID; // Class identification, replacement for typeinfo
+
+  DXILMetadataAnalysisWrapperPass();
+  ~DXILMetadataAnalysisWrapperPass() override;
+
+  const dxil::ModuleMetadataInfo &getModuleMetadata() const {
+    return *MetadataInfo;
+  }
+  dxil::ModuleMetadataInfo &getModuleMetadata() { return *MetadataInfo; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnModule(Module &M) override;
+  void releaseMemory() override;
+
+  void print(raw_ostream &OS, const Module *M) const override;
+  void dump() const;
+};
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_DXILMETADATA_H
diff --git a/llvm/include/llvm/BinaryFormat/Minidump.h b/llvm/include/llvm/BinaryFormat/Minidump.h
index 966930325261596..8054e81322a92a3 100644
--- a/llvm/include/llvm/BinaryFormat/Minidump.h
+++ b/llvm/include/llvm/BinaryFormat/Minidump.h
@@ -74,6 +74,18 @@ struct MemoryDescriptor_64 {
   support::ulittle64_t StartOfMemoryRange;
   support::ulittle64_t DataSize;
 };
+static_assert(sizeof(MemoryDescriptor_64) == 16);
+
+struct MemoryListHeader {
+  support::ulittle32_t NumberOfMemoryRanges;
+};
+static_assert(sizeof(MemoryListHeader) == 4);
+
+struct Memory64ListHeader {
+  support::ulittle64_t NumberOfMemoryRanges;
+  support::ulittle64_t BaseRVA;
+};
+static_assert(sizeof(Memory64ListHeader) == 16);
 
 struct MemoryInfoListHeader {
   support::ulittle32_t SizeOfHeader;
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 3a4e081831e3154..deb1d04df3400cd 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3156,8 +3156,11 @@ class TargetLoweringBase {
   ///
   /// \p DI is the deinterleave intrinsic.
   /// \p LI is the accompanying load instruction
-  virtual bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *DI,
-                                                LoadInst *LI) const {
+  /// \p DeadInsts is a reference to a vector that keeps track of dead
+  /// instruction during transformations.
+  virtual bool lowerDeinterleaveIntrinsicToLoad(
+      IntrinsicInst *DI, LoadInst *LI,
+      SmallVectorImpl<Instruction *> &DeadInsts) const {
     return false;
   }
 
@@ -3167,8 +3170,11 @@ class TargetLoweringBase {
   ///
   /// \p II is the interleave intrinsic.
   /// \p SI is the accompanying store instruction
-  virtual bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II,
-                                               StoreInst *SI) const {
+  /// \p DeadInsts is a reference to a vector that keeps track of dead
+  /// instruction during transformations.
+  virtual bool lowerInterleaveIntrinsicToStore(
+      IntrinsicInst *II, StoreInst *SI,
+      SmallVectorImpl<Instruction *> &DeadInsts) const {
     return false;
   }
 
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index efd2ff63bda54f5..111384b302df68e 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -16,7 +16,7 @@
 
 /* Indicate that this is LLVM compiled from the amd-gfx branch. */
 #define LLVM_HAVE_BRANCH_AMD_GFX
-#define LLVM_MAIN_REVISION 508173
+#define LLVM_MAIN_REVISION 508242
 
 /* Define if LLVM_ENABLE_DUMP is enabled */
 #cmakedefine LLVM_ENABLE_DUMP
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 9c70a60bf50c775..8c6b7895470b8dd 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -2940,6 +2940,17 @@ inline VScaleVal_match m_VScale() {
   return VScaleVal_match();
 }
 
+template <typename Opnd0, typename Opnd1>
+inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty
+m_Interleave2(const Opnd0 &Op0, const Opnd1 &Op1) {
+  return m_Intrinsic<Intrinsic::vector_interleave2>(Op0, Op1);
+}
+
+template <typename Opnd>
+inline typename m_Intrinsic_Ty<Opnd>::Ty m_Deinterleave2(const Opnd &Op) {
+  return m_Intrinsic<Intrinsic::vector_deinterleave2>(Op);
+}
+
 template <typename LHS, typename RHS, unsigned Opcode, bool Commutable = false>
 struct LogicalOp_match {
   LHS L;
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 5569740a27cf73b..652a78cb7b426e2 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -83,6 +83,8 @@ void initializeCycleInfoWrapperPassPass(PassRegistry &);
 void initializeDAEPass(PassRegistry&);
 void initializeDAHPass(PassRegistry&);
 void initializeDCELegacyPassPass(PassRegistry&);
+void initializeDXILMetadataAnalysisWrapperPassPass(PassRegistry &);
+void initializeDXILMetadataAnalysisWrapperPrinterPass(PassRegistry &);
 void initializeDeadMachineInstructionElimPass(PassRegistry&);
 void initializeDebugifyMachineModulePass(PassRegistry &);
 void initializeDependenceAnalysisWrapperPassPass(PassRegistry&);
diff --git a/llvm/include/llvm/Object/Minidump.h b/llvm/include/llvm/Object/Minidump.h
index e45d4de0090def7..65ad6b171c2dc1a 100644
--- a/llvm/include/llvm/Object/Minidump.h
+++ b/llvm/include/llvm/Object/Minidump.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/fallible_iterator.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/Minidump.h"
 #include "llvm/Object/Binary.h"
@@ -103,6 +104,13 @@ class MinidumpFile : public Binary {
         minidump::StreamType::MemoryList);
   }
 
+  /// Returns the header to the memory 64 list stream. An error is returned if
+  /// the file does not contain this stream.
+  Expected<minidump::Memory64ListHeader> getMemoryList64Header() const {
+    return getStream<minidump::Memory64ListHeader>(
+        minidump::StreamType::Memory64List);
+  }
+
   class MemoryInfoIterator
       : public iterator_facade_base<MemoryInfoIterator,
                                     std::forward_iterator_tag,
@@ -132,6 +140,90 @@ class MinidumpFile : public Binary {
     size_t Stride;
   };
 
+  /// Class the provides an iterator over the memory64 memory ranges. Only the
+  /// the first descriptor is validated as readable beforehand.
+  class Memory64Iterator {
+  public:
+    static Memory64Iterator
+    begin(ArrayRef<uint8_t> Storage,
+          ArrayRef<minidump::MemoryDescriptor_64> Descriptors) {
+      return Memory64Iterator(Storage, Descriptors);
+    }
+
+    static Memory64Iterator end() { return Memory64Iterator(); }
+
+    bool operator==(const Memory64Iterator &R) const {
+      return IsEnd == R.IsEnd;
+    }
+
+    bool operator!=(const Memory64Iterator &R) const { return !(*this == R); }
+
+    const std::pair<minidump::MemoryDescriptor_64, ArrayRef<uint8_t>> &
+    operator*() {
+      return Current;
+    }
+
+    const std::pair<minidump::MemoryDescriptor_64, ArrayRef<uint8_t>> *
+    operator->() {
+      return &Current;
+    }
+
+    Error inc() {
+      if (Descriptors.empty()) {
+        IsEnd = true;
+        return Error::success();
+      }
+
+      // Drop front gives us an array ref, so we need to call .front() as well.
+      const minidump::MemoryDescriptor_64 &Descriptor = Descriptors.front();
+      if (Descriptor.DataSize > Storage.size()) {
+        IsEnd = true;
+        return make_error<GenericBinaryError>(
+            "Memory64 Descriptor exceeds end of file.",
+            object_error::unexpected_eof);
+      }
+
+      ArrayRef<uint8_t> Content = Storage.take_front(Descriptor.DataSize);
+      Current = std::make_pair(Descriptor, Content);
+
+      Storage = Storage.drop_front(Descriptor.DataSize);
+      Descriptors = Descriptors.drop_front();
+
+      return Error::success();
+    }
+
+  private:
+    // This constructor expects that the first descriptor is readable.
+    Memory64Iterator(ArrayRef<uint8_t> Storage,
+                     ArrayRef<minidump::MemoryDescriptor_64> Descriptors)
+        : Storage(Storage), Descriptors(Descriptors), IsEnd(false) {
+      assert(!Descriptors.empty() &&
+             Storage.size() >= Descriptors.front().DataSize);
+      minidump::MemoryDescriptor_64 Descriptor = Descriptors.front();
+      ArrayRef<uint8_t> Content = Storage.take_front(Descriptor.DataSize);
+      Current = std::make_pair(Descriptor, Content);
+      this->Descriptors = Descriptors.drop_front();
+      this->Storage = Storage.drop_front(Descriptor.DataSize);
+    }
+
+    Memory64Iterator()
+        : Storage(ArrayRef<uint8_t>()),
+          Descriptors(ArrayRef<minidump::MemoryDescriptor_64>()), IsEnd(true) {}
+
+    std::pair<minidump::MemoryDescriptor_64, ArrayRef<uint8_t>> Current;
+    ArrayRef<uint8_t> Storage;
+    ArrayRef<minidump::MemoryDescriptor_64> Descriptors;
+    bool IsEnd;
+  };
+
+  using FallibleMemory64Iterator = llvm::fallible_iterator<Memory64Iterator>;
+
+  /// Returns an iterator that pairs each descriptor with it's respective
+  /// content from the Memory64List stream. An error is returned if the file
+  /// does not contain a Memory64List stream, or if the descriptor data is
+  /// unreadable.
+  iterator_range<FallibleMemory64Iterator> getMemory64List(Error &Err) const;
+
   /// Returns the list of descriptors embedded in the MemoryInfoList stream. The
   /// descriptors provide properties (e.g. permissions) of interesting regions
   /// of memory at the time the minidump was taken. An error is returned if the
@@ -152,15 +244,15 @@ class MinidumpFile : public Binary {
   }
 
   /// Return a slice of the given data array, with bounds checking.
-  static Expected<ArrayRef<uint8_t>> getDataSlice(ArrayRef<uint8_t> Data,
-                                                  size_t Offset, size_t Size);
+  static Expected<ArrayRef<uint8_t>>
+  getDataSlice(ArrayRef<uint8_t> Data, uint64_t Offset, uint64_t Size);
 
   /// Return the slice of the given data array as an array of objects of the
   /// given type. The function checks that the input array is large enough to
   /// contain the correct number of objects of the given type.
   template <typename T>
   static Expected<ArrayRef<T>> getDataSliceAs(ArrayRef<uint8_t> Data,
-                                              size_t Offset, size_t Count);
+                                              uint64_t Offset, uint64_t Count);
 
   MinidumpFile(MemoryBufferRef Source, const minidump::Header &Header,
                ArrayRef<minidump::Directory> Streams,
@@ -199,15 +291,16 @@ Expected<const T &> MinidumpFile::getStream(minidump::StreamType Type) const {
 
 template <typename T>
 Expected<ArrayRef<T>> MinidumpFile::getDataSliceAs(ArrayRef<uint8_t> Data,
-                                                   size_t Offset,
-                                                   size_t Count) {
+                                                   uint64_t Offset,
+                                                   uint64_t Count) {
   // Check for overflow.
-  if (Count > std::numeric_limits<size_t>::max() / sizeof(T))
+  if (Count > std::numeric_limits<uint64_t>::max() / sizeof(T))
     return createEOFError();
   Expected<ArrayRef<uint8_t>> Slice =
       getDataSlice(Data, Offset, sizeof(T) * Count);
   if (!Slice)
     return Slice.takeError();
+
   return ArrayRef<T>(reinterpret_cast<const T *>(Slice->data()), Count);
 }
 
diff --git a/llvm/include/llvm/ObjectYAML/MinidumpYAML.h b/llvm/include/llvm/ObjectYAML/MinidumpYAML.h
index b0cee541cef206a..02371e6d867206e 100644
--- a/llvm/include/llvm/ObjectYAML/MinidumpYAML.h
+++ b/llvm/include/llvm/ObjectYAML/MinidumpYAML.h
@@ -29,6 +29,7 @@ struct Stream {
     Exception,
     MemoryInfoList,
     MemoryList,
+    Memory64List,
     ModuleList,
     RawContent,
     SystemInfo,
@@ -98,12 +99,30 @@ struct ParsedMemoryDescriptor {
   minidump::MemoryDescriptor Entry;
   yaml::BinaryRef Content;
 };
+
+struct ParsedMemory64Descriptor {
+  static constexpr Stream::StreamKind Kind = Stream::StreamKind::Memory64List;
+  static constexpr minidump::StreamType Type =
+      minidump::StreamType::Memory64List;
+
+  minidump::MemoryDescriptor_64 Entry;
+  yaml::BinaryRef Content;
+};
 } // namespace detail
 
 using ModuleListStream = detail::ListStream<detail::ParsedModule>;
 using ThreadListStream = detail::ListStream<detail::ParsedThread>;
 using MemoryListStream = detail::ListStream<detail::ParsedMemoryDescriptor>;
 
+struct Memory64ListStream
+    : public detail::ListStream<detail::ParsedMemory64Descriptor> {
+  minidump::Memory64ListHeader Header;
+
+  explicit Memory64ListStream(
+      std::vector<detail::ParsedMemory64Descriptor> Entries = {})
+      : ListStream(Entries) {}
+};
+
 /// ExceptionStream minidump stream.
 struct ExceptionStream : public Stream {
   minidump::ExceptionStream MDExceptionStream;
@@ -244,6 +263,12 @@ template <> struct MappingContextTraits<minidump::MemoryDescriptor, BinaryRef> {
                       BinaryRef &Content);
 };
 
+template <>
+struct MappingContextTraits<minidump::MemoryDescriptor_64, BinaryRef> {
+  static void mapping(IO &IO, minidump::MemoryDescriptor_64 &Memory,
+                      BinaryRef &Content);
+};
+
 } // namespace yaml
 
 } // namespace llvm
@@ -269,11 +294,14 @@ LLVM_YAML_DECLARE_MAPPING_TRAITS(
     llvm::MinidumpYAML::ModuleListStream::entry_type)
 LLVM_YAML_DECLARE_MAPPING_TRAITS(
     llvm::MinidumpYAML::ThreadListStream::entry_type)
+LLVM_YAML_DECLARE_MAPPING_TRAITS(
+    llvm::MinidumpYAML::Memory64ListStream::entry_type)
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::MinidumpYAML::Stream>)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::MemoryListStream::entry_type)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ModuleListStream::entry_type)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ThreadListStream::entry_type)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::Memory64ListStream::entry_type)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::minidump::MemoryInfo)
 
 LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::MinidumpYAML::Object)
diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index cf1246951ecc183..c160520788d873b 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -111,6 +111,7 @@ class Context;
 class Function;
 class Instruction;
 class SelectInst;
+class ExtractElementInst;
 class InsertElementInst;
 class BranchInst;
 class UnaryInstruction;
@@ -232,24 +233,25 @@ class Value {
   /// order.
   llvm::Value *Val = nullptr;
 
-  friend class Context;           // For getting `Val`.
-  friend class User;              // For getting `Val`.
-  friend class Use;               // For getting `Val`.
-  friend class SelectInst;        // For getting `Val`.
-  friend class InsertElementInst; // For getting `Val`.
-  friend class BranchInst;        // For getting `Val`.
-  friend class LoadInst;          // For getting `Val`.
-  friend class StoreInst;         // For getting `Val`.
-  friend class ReturnInst;        // For getting `Val`.
-  friend class CallBase;          // For getting `Val`.
-  friend class CallInst;          // For getting `Val`.
-  friend class InvokeInst;        // For getting `Val`.
-  friend class CallBrInst;        // For getting `Val`.
-  friend class GetElementPtrInst; // For getting `Val`.
-  friend class AllocaInst;        // For getting `Val`.
-  friend class CastInst;          // For getting `Val`.
-  friend class PHINode;           // For getting `Val`.
-  friend class UnreachableInst;   // For getting `Val`.
+  friend class Context;            // For getting `Val`.
+  friend class User;               // For getting `Val`.
+  friend class Use;                // For getting `Val`.
+  friend class SelectInst;         // For getting `Val`.
+  friend class ExtractElementInst; // For getting `Val`.
+  friend class InsertElementInst;  // For getting `Val`.
+  friend class BranchInst;         // For getting `Val`.
+  friend class LoadInst;           // For getting `Val`.
+  friend class StoreInst;          // For getting `Val`.
+  friend class ReturnInst;         // For getting `Val`.
+  friend class CallBase;           // For getting `Val`.
+  friend class CallInst;           // For getting `Val`.
+  friend class InvokeInst;         // For getting `Val`.
+  friend class CallBrInst;         // For getting `Val`.
+  friend class GetElementPtrInst;  // For getting `Val`.
+  friend class AllocaInst;         // For getting `Val`.
+  friend class CastInst;           // For getting `Val`.
+  friend class PHINode;            // For getting `Val`.
+  friend class UnreachableInst;    // For getting `Val`.
 
   /// All values point to the context.
   Context &Ctx;
@@ -615,20 +617,21 @@ class Instruction : public sandboxir::User {
   /// A SandboxIR Instruction may map to multiple LLVM IR Instruction. This
   /// returns its topmost LLVM IR instruction.
   llvm::Instruction *getTopmostLLVMInstruction() const;
-  friend class SelectInst;        // For getTopmostLLVMInstruction().
-  friend class InsertElementInst; // For getTopmostLLVMInstruction().
-  friend class BranchInst;        // For getTopmostLLVMInstruction().
-  friend class LoadInst;          // For getTopmostLLVMInstruction().
-  friend class StoreInst;         // For getTopmostLLVMInstruction().
-  friend class ReturnInst;        // For getTopmostLLVMInstruction().
-  friend class CallInst;          // For getTopmostLLVMInstruction().
-  friend class InvokeInst;        // For getTopmostLLVMInstruction().
-  friend class CallBrInst;        // For getTopmostLLVMInstruction().
-  friend class GetElementPtrInst; // For getTopmostLLVMInstruction().
-  friend class AllocaInst;        // For getTopmostLLVMInstruction().
-  friend class CastInst;          // For getTopmostLLVMInstruction().
-  friend class PHINode;           // For getTopmostLLVMInstruction().
-  friend class UnreachableInst;   // For getTopmostLLVMInstruction().
+  friend class SelectInst;         // For getTopmostLLVMInstruction().
+  friend class ExtractElementInst; // For getTopmostLLVMInstruction().
+  friend class InsertElementInst;  // For getTopmostLLVMInstruction().
+  friend class BranchInst;         // For getTopmostLLVMInstruction().
+  friend class LoadInst;           // For getTopmostLLVMInstruction().
+  friend class StoreInst;          // For getTopmostLLVMInstruction().
+  friend class ReturnInst;         // For getTopmostLLVMInstruction().
+  friend class CallInst;           // For getTopmostLLVMInstruction().
+  friend class InvokeInst;         // For getTopmostLLVMInstruction().
+  friend class CallBrInst;         // For getTopmostLLVMInstruction().
+  friend class GetElementPtrInst;  // For getTopmostLLVMInstruction().
+  friend class AllocaInst;         // For getTopmostLLVMInstruction().
+  friend class CastInst;           // For getTopmostLLVMInstruction().
+  friend class PHINode;            // For getTopmostLLVMInstruction().
+  friend class UnreachableInst;    // For getTopmostLLVMInstruction().
 
   /// \Returns the LLVM IR Instructions that this SandboxIR maps to in program
   /// order.
@@ -768,6 +771,37 @@ class InsertElementInst final
   }
 };
 
+class ExtractElementInst final
+    : public SingleLLVMInstructionImpl<llvm::ExtractElementInst> {
+  /// Use Context::createExtractElementInst() instead.
+  ExtractElementInst(llvm::Instruction *I, Context &Ctx)
+      : SingleLLVMInstructionImpl(ClassID::ExtractElement,
+                                  Opcode::ExtractElement, I, Ctx) {}
+  friend class Context; // For accessing the constructor in
+                        // create*()
+
+public:
+  static Value *create(Value *Vec, Value *Idx, Instruction *InsertBefore,
+                       Context &Ctx, const Twine &Name = "");
+  static Value *create(Value *Vec, Value *Idx, BasicBlock *InsertAtEnd,
+                       Context &Ctx, const Twine &Name = "");
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::ExtractElement;
+  }
+
+  static bool isValidOperands(const Value *Vec, const Value *Idx) {
+    return llvm::ExtractElementInst::isValidOperands(Vec->Val, Idx->Val);
+  }
+  Value *getVectorOperand() { return getOperand(0); }
+  Value *getIndexOperand() { return getOperand(1); }
+  const Value *getVectorOperand() const { return getOperand(0); }
+  const Value *getIndexOperand() const { return getOperand(1); }
+
+  VectorType *getVectorOperandType() const {
+    return cast<VectorType>(getVectorOperand()->getType());
+  }
+};
+
 class BranchInst : public SingleLLVMInstructionImpl<llvm::BranchInst> {
   /// Use Context::createBranchInst(). Don't call the constructor directly.
   BranchInst(llvm::BranchInst *BI, Context &Ctx)
@@ -1644,6 +1678,8 @@ class Context {
   friend SelectInst; // For createSelectInst()
   InsertElementInst *createInsertElementInst(llvm::InsertElementInst *IEI);
   friend InsertElementInst; // For createInsertElementInst()
+  ExtractElementInst *createExtractElementInst(llvm::ExtractElementInst *EEI);
+  friend ExtractElementInst; // For createExtractElementInst()
   BranchInst *createBranchInst(llvm::BranchInst *I);
   friend BranchInst; // For createBranchInst()
   LoadInst *createLoadInst(llvm::LoadInst *LI);
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index 269aea784dcec17..11f4f2e74712f4e 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -32,36 +32,37 @@ DEF_USER(Constant, Constant)
 #define OPCODES(...)
 #endif
 // clang-format off
-//       ClassID,        Opcode(s),         Class
-DEF_INSTR(Opaque,        OP(Opaque),        OpaqueInst)
-DEF_INSTR(InsertElement, OP(InsertElement), InsertElementInst)
-DEF_INSTR(Select,        OP(Select),        SelectInst)
-DEF_INSTR(Br,            OP(Br),            BranchInst)
-DEF_INSTR(Load,          OP(Load),          LoadInst)
-DEF_INSTR(Store,         OP(Store),         StoreInst)
-DEF_INSTR(Ret,           OP(Ret),           ReturnInst)
-DEF_INSTR(Call,          OP(Call),          CallInst)
-DEF_INSTR(Invoke,        OP(Invoke),        InvokeInst)
-DEF_INSTR(CallBr,        OP(CallBr),        CallBrInst)
-DEF_INSTR(GetElementPtr, OP(GetElementPtr), GetElementPtrInst)
-DEF_INSTR(Alloca,        OP(Alloca),        AllocaInst)
-DEF_INSTR(Cast,  OPCODES(\
-                         OP(ZExt)          \
-                         OP(SExt)          \
-                         OP(FPToUI)        \
-                         OP(FPToSI)        \
-                         OP(FPExt)         \
-                         OP(PtrToInt)      \
-                         OP(IntToPtr)      \
-                         OP(SIToFP)        \
-                         OP(UIToFP)        \
-                         OP(Trunc)         \
-                         OP(FPTrunc)       \
-                         OP(BitCast)       \
-                         OP(AddrSpaceCast) \
-                         ),                 CastInst)
-DEF_INSTR(PHI,           OP(PHI),           PHINode)
-DEF_INSTR(Unreachable,   OP(Unreachable),   UnreachableInst)
+//        ClassID,        Opcode(s),          Class
+DEF_INSTR(Opaque,         OP(Opaque),         OpaqueInst)
+DEF_INSTR(ExtractElement, OP(ExtractElement), ExtractElementInst)
+DEF_INSTR(InsertElement,  OP(InsertElement),  InsertElementInst)
+DEF_INSTR(Select,         OP(Select),         SelectInst)
+DEF_INSTR(Br,             OP(Br),             BranchInst)
+DEF_INSTR(Load,           OP(Load),           LoadInst)
+DEF_INSTR(Store,          OP(Store),          StoreInst)
+DEF_INSTR(Ret,            OP(Ret),            ReturnInst)
+DEF_INSTR(Call,           OP(Call),           CallInst)
+DEF_INSTR(Invoke,         OP(Invoke),         InvokeInst)
+DEF_INSTR(CallBr,         OP(CallBr),         CallBrInst)
+DEF_INSTR(GetElementPtr,  OP(GetElementPtr),  GetElementPtrInst)
+DEF_INSTR(Alloca,         OP(Alloca),         AllocaInst)
+DEF_INSTR(Cast,   OPCODES(\
+                          OP(ZExt)          \
+                          OP(SExt)          \
+                          OP(FPToUI)        \
+                          OP(FPToSI)        \
+                          OP(FPExt)         \
+                          OP(PtrToInt)      \
+                          OP(IntToPtr)      \
+                          OP(SIToFP)        \
+                          OP(UIToFP)        \
+                          OP(Trunc)         \
+                          OP(FPTrunc)       \
+                          OP(BitCast)       \
+                          OP(AddrSpaceCast) \
+                          ),                  CastInst)
+DEF_INSTR(PHI,            OP(PHI),            PHINode)
+DEF_INSTR(Unreachable,    OP(Unreachable),    UnreachableInst)
 
 // clang-format on
 #ifdef DEF_VALUE
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index fd47ddda6ca3a5c..4c04d81ace82743 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -1079,6 +1079,13 @@ def zanyext : PatFrags<(ops node:$op),
                        [(zext node:$op),
                         (anyext node:$op)]>;
 
+def zext_nneg : PatFrag<(ops node:$src), (zext node:$src), [{
+  return N->getFlags().hasNonNeg();
+}]>;
+def sext_like : PatFrags<(ops node:$src),
+                         [(zext_nneg node:$src),
+                          (sext node:$src)]>;
+
 // null_frag - The null pattern operator is used in multiclass instantiations
 // which accept an SDPatternOperator for use in matching patterns for internal
 // definitions. When expanding a pattern, if the null fragment is referenced
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 2cb3547ec40473c..393803fad89383f 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -61,6 +61,7 @@ add_llvm_component_library(LLVMAnalysis
   DomTreeUpdater.cpp
   DominanceFrontier.cpp
   DXILResource.cpp
+  DXILMetadataAnalysis.cpp
   FunctionPropertiesAnalysis.cpp
   GlobalsModRef.cpp
   GuardUtils.cpp
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 084647b1d9d938a..defcacdfa8b105a 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -3400,8 +3400,9 @@ ConstantFoldScalarFrexpCall(Constant *Op, Type *IntTy) {
 
   // The exponent is an "unspecified value" for inf/nan. We use zero to avoid
   // using undef.
-  Constant *Result1 = FrexpMant.isFinite() ? ConstantInt::get(IntTy, FrexpExp)
-                                           : ConstantInt::getNullValue(IntTy);
+  Constant *Result1 = FrexpMant.isFinite()
+                          ? ConstantInt::getSigned(IntTy, FrexpExp)
+                          : ConstantInt::getNullValue(IntTy);
   return {Result0, Result1};
 }
 
diff --git a/llvm/lib/Analysis/DXILMetadataAnalysis.cpp b/llvm/lib/Analysis/DXILMetadataAnalysis.cpp
new file mode 100644
index 000000000000000..202beeea841c11a
--- /dev/null
+++ b/llvm/lib/Analysis/DXILMetadataAnalysis.cpp
@@ -0,0 +1,96 @@
+//=- DXILMetadataAnalysis.cpp - Representation of Module metadata -*- C++ -*=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/DXILMetadataAnalysis.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+
+#define DEBUG_TYPE "dxil-metadata-analysis"
+
+using namespace llvm;
+using namespace dxil;
+
+static ModuleMetadataInfo collectMetadataInfo(Module &M) {
+  ModuleMetadataInfo MMDAI;
+  Triple TT(Triple(M.getTargetTriple()));
+  MMDAI.DXILVersion = TT.getDXILVersion();
+  MMDAI.ShaderModelVersion = TT.getOSVersion();
+  MMDAI.ShaderStage = TT.getEnvironment();
+  return MMDAI;
+}
+
+void ModuleMetadataInfo::print(raw_ostream &OS) const {
+  OS << "Shader Model Version : " << ShaderModelVersion.getAsString() << "\n";
+  OS << "DXIL Version : " << DXILVersion.getAsString() << "\n";
+  OS << "Shader Stage : " << Triple::getEnvironmentTypeName(ShaderStage)
+     << "\n";
+}
+
+//===----------------------------------------------------------------------===//
+// DXILMetadataAnalysis and DXILMetadataAnalysisPrinterPass
+
+// Provide an explicit template instantiation for the static ID.
+AnalysisKey DXILMetadataAnalysis::Key;
+
+llvm::dxil::ModuleMetadataInfo
+DXILMetadataAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
+  return collectMetadataInfo(M);
+}
+
+PreservedAnalyses
+DXILMetadataAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &AM) {
+  llvm::dxil::ModuleMetadataInfo &Data = AM.getResult<DXILMetadataAnalysis>(M);
+
+  Data.print(OS);
+  return PreservedAnalyses::all();
+}
+
+//===----------------------------------------------------------------------===//
+// DXILMetadataAnalysisWrapperPass
+
+DXILMetadataAnalysisWrapperPass::DXILMetadataAnalysisWrapperPass()
+    : ModulePass(ID) {
+  initializeDXILMetadataAnalysisWrapperPassPass(
+      *PassRegistry::getPassRegistry());
+}
+
+DXILMetadataAnalysisWrapperPass::~DXILMetadataAnalysisWrapperPass() = default;
+
+void DXILMetadataAnalysisWrapperPass::getAnalysisUsage(
+    AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+}
+
+bool DXILMetadataAnalysisWrapperPass::runOnModule(Module &M) {
+  MetadataInfo.reset(new ModuleMetadataInfo(collectMetadataInfo(M)));
+  return false;
+}
+
+void DXILMetadataAnalysisWrapperPass::releaseMemory() { MetadataInfo.reset(); }
+
+void DXILMetadataAnalysisWrapperPass::print(raw_ostream &OS,
+                                            const Module *) const {
+  if (!MetadataInfo) {
+    OS << "No module metadata info has been built!\n";
+    return;
+  }
+  MetadataInfo->print(dbgs());
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD
+void DXILMetadataAnalysisWrapperPass::dump() const { print(dbgs(), nullptr); }
+#endif
+
+INITIALIZE_PASS(DXILMetadataAnalysisWrapperPass, "dxil-metadata-analysis",
+                "DXIL Module Metadata analysis", false, true)
+char DXILMetadataAnalysisWrapperPass::ID = 0;
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 318a9d773dc2e18..de220d68a59163b 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -9136,11 +9136,12 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
       }
 
   // If this loop must exit based on this condition (or execute undefined
-  // behaviour), and we can prove the test sequence produced must repeat
-  // the same values on self-wrap of the IV, then we can infer that IV
-  // doesn't self wrap because if it did, we'd have an infinite (undefined)
-  // loop.  Note that a stride of 0 is trivially no-self-wrap by definition.
+  // behaviour), see if we can improve wrap flags.  This is essentially
+  // a must execute style proof.
   if (ControllingFiniteLoop && isLoopInvariant(RHS, L)) {
+    // If we can prove the test sequence produced must repeat the same values
+    // on self-wrap of the IV, then we can infer that IV doesn't self wrap
+    // because if it did, we'd have an infinite (undefined) loop.
     // TODO: We can peel off any functions which are invertible *in L*.  Loop
     // invariant terms are effectively constants for our purposes here.
     auto *InnerLHS = LHS;
@@ -9156,6 +9157,25 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
       Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
       setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
     }
+
+    // For a slt/ult condition with a positive step, can we prove nsw/nuw?
+    // From no-self-wrap, this follows trivially from the fact that every
+    // (un)signed-wrapped, but not self-wrapped value must be LT than the
+    // last value before (un)signed wrap.  Since we know that last value
+    // didn't exit, nor will any smaller one.
+    if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
+      auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
+      if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS);
+          AR && AR->getLoop() == L && AR->isAffine() &&
+          !AR->getNoWrapFlags(WrapType) && AR->hasNoSelfWrap() &&
+          isKnownPositive(AR->getStepRecurrence(*this))) {
+        auto Flags = AR->getNoWrapFlags();
+        Flags = setFlags(Flags, WrapType);
+        SmallVector<const SCEV*> Operands{AR->operands()};
+        Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
+        setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), Flags);
+      }
+    }
   }
 
   switch (Pred) {
@@ -12769,35 +12789,6 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
 
   const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS);
   bool PredicatedIV = false;
-
-  auto canAssumeNoSelfWrap = [&](const SCEVAddRecExpr *AR) {
-    // Can we prove this loop *must* be UB if overflow of IV occurs?
-    // Reasoning goes as follows:
-    // * Suppose the IV did self wrap.
-    // * If Stride evenly divides the iteration space, then once wrap
-    //   occurs, the loop must revisit the same values.
-    // * We know that RHS is invariant, and that none of those values
-    //   caused this exit to be taken previously.  Thus, this exit is
-    //   dynamically dead.
-    // * If this is the sole exit, then a dead exit implies the loop
-    //   must be infinite if there are no abnormal exits.
-    // * If the loop were infinite, then it must either not be mustprogress
-    //   or have side effects. Otherwise, it must be UB.
-    // * It can't (by assumption), be UB so we have contradicted our
-    //   premise and can conclude the IV did not in fact self-wrap.
-    if (!isLoopInvariant(RHS, L))
-      return false;
-
-    if (!isKnownToBeAPowerOfTwo(AR->getStepRecurrence(*this), /*OrZero=*/true,
-                                /*OrNegative*/ true))
-      return false;
-
-    if (!ControlsOnlyExit || !loopHasNoAbnormalExits(L))
-      return false;
-
-    return loopIsFiniteByAssumption(L);
-  };
-
   if (!IV) {
     if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(LHS)) {
       const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(ZExt->getOperand());
@@ -12948,21 +12939,10 @@ ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
         Stride = getUMaxExpr(Stride, getOne(Stride->getType()));
       }
     }
-  } else if (!Stride->isOne() && !NoWrap) {
-    auto isUBOnWrap = [&]() {
-      // From no-self-wrap, we need to then prove no-(un)signed-wrap.  This
-      // follows trivially from the fact that every (un)signed-wrapped, but
-      // not self-wrapped value must be LT than the last value before
-      // (un)signed wrap.  Since we know that last value didn't exit, nor
-      // will any smaller one.
-      return canAssumeNoSelfWrap(IV);
-    };
-
+  } else if (!NoWrap) {
     // Avoid proven overflow cases: this will ensure that the backedge taken
-    // count will not generate any unsigned overflow. Relaxed no-overflow
-    // conditions exploit NoWrapFlags, allowing to optimize in presence of
-    // undefined behaviors like the case of C language.
-    if (canIVOverflowOnLT(RHS, Stride, IsSigned) && !isUBOnWrap())
+    // count will not generate any unsigned overflow.
+    if (canIVOverflowOnLT(RHS, Stride, IsSigned))
       return getCouldNotCompute();
   }
 
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 22d0708f5478604..1fb37fb8406ef45 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -1643,7 +1643,7 @@ static bool matchUAddWithOverflowConstantEdgeCases(CmpInst *Cmp,
   if (Pred == ICmpInst::ICMP_EQ && match(B, m_AllOnes()))
     B = ConstantInt::get(B->getType(), 1);
   else if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt()))
-    B = ConstantInt::get(B->getType(), -1);
+    B = Constant::getAllOnesValue(B->getType());
   else
     return false;
 
diff --git a/llvm/lib/CodeGen/ExpandMemCmp.cpp b/llvm/lib/CodeGen/ExpandMemCmp.cpp
index 2758f7be4d505f3..04222d5b4afd4c1 100644
--- a/llvm/lib/CodeGen/ExpandMemCmp.cpp
+++ b/llvm/lib/CodeGen/ExpandMemCmp.cpp
@@ -590,7 +590,7 @@ void MemCmpExpansion::emitMemCmpResultBlock() {
                                   ResBlock.PhiSrc2);
 
   Value *Res =
-      Builder.CreateSelect(Cmp, ConstantInt::get(Builder.getInt32Ty(), -1),
+      Builder.CreateSelect(Cmp, Constant::getAllOnesValue(Builder.getInt32Ty()),
                            ConstantInt::get(Builder.getInt32Ty(), 1));
 
   PhiRes->addIncoming(Res, ResBlock.BB);
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index 8c9065aec7faa4f..ef11713892a53d7 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -489,7 +489,7 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
   LLVM_DEBUG(dbgs() << "IA: Found a deinterleave intrinsic: " << *DI << "\n");
 
   // Try and match this with target specific intrinsics.
-  if (!TLI->lowerDeinterleaveIntrinsicToLoad(DI, LI))
+  if (!TLI->lowerDeinterleaveIntrinsicToLoad(DI, LI, DeadInsts))
     return false;
 
   // We now have a target-specific load, so delete the old one.
@@ -510,13 +510,16 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
 
   LLVM_DEBUG(dbgs() << "IA: Found an interleave intrinsic: " << *II << "\n");
 
+  SmallVector<Instruction *, 4> InterleaveDeadInsts;
   // Try and match this with target specific intrinsics.
-  if (!TLI->lowerInterleaveIntrinsicToStore(II, SI))
+  if (!TLI->lowerInterleaveIntrinsicToStore(II, SI, InterleaveDeadInsts))
     return false;
 
   // We now have a target-specific store, so delete the old one.
   DeadInsts.push_back(SI);
   DeadInsts.push_back(II);
+  DeadInsts.insert(DeadInsts.end(), InterleaveDeadInsts.begin(),
+                   InterleaveDeadInsts.end());
   return true;
 }
 
@@ -537,7 +540,7 @@ bool InterleavedAccessImpl::runOnFunction(Function &F) {
       // with a factor of 2.
       if (II->getIntrinsicID() == Intrinsic::vector_deinterleave2)
         Changed |= lowerDeinterleaveIntrinsic(II, DeadInsts);
-      if (II->getIntrinsicID() == Intrinsic::vector_interleave2)
+      else if (II->getIntrinsicID() == Intrinsic::vector_interleave2)
         Changed |= lowerInterleaveIntrinsic(II, DeadInsts);
     }
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 3635bc7a9658049..5b66748297b7ef4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -1056,31 +1056,38 @@ SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
     return matcher.getNode(ISD::USUBSAT, dl, Op1.getValueType(), Op1, Op2);
   }
 
+  if (Opcode == ISD::UADDSAT) {
+    EVT OVT = Op1.getValueType();
+    EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
+    // We can promote if we use sign-extend. Do this if the target prefers.
+    if (TLI.isSExtCheaperThanZExt(OVT, NVT)) {
+      Op1 = SExtPromotedInteger(Op1);
+      Op2 = SExtPromotedInteger(Op2);
+      return matcher.getNode(ISD::UADDSAT, dl, NVT, Op1, Op2);
+    }
+
+    Op1 = ZExtPromotedInteger(Op1);
+    Op2 = ZExtPromotedInteger(Op2);
+    unsigned NewBits = NVT.getScalarSizeInBits();
+    APInt MaxVal = APInt::getLowBitsSet(NewBits, OldBits);
+    SDValue SatMax = DAG.getConstant(MaxVal, dl, NVT);
+    SDValue Add = matcher.getNode(ISD::ADD, dl, NVT, Op1, Op2);
+    return matcher.getNode(ISD::UMIN, dl, NVT, Add, SatMax);
+  }
+
   bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
 
   // FIXME: We need vp-aware PromotedInteger functions.
-  SDValue Op1Promoted, Op2Promoted;
   if (IsShift) {
-    Op1Promoted = GetPromotedInteger(Op1);
-    Op2Promoted = ZExtPromotedInteger(Op2);
-  } else if (Opcode == ISD::UADDSAT) {
-    Op1Promoted = ZExtPromotedInteger(Op1);
-    Op2Promoted = ZExtPromotedInteger(Op2);
+    Op1 = GetPromotedInteger(Op1);
+    Op2 = ZExtPromotedInteger(Op2);
   } else {
-    Op1Promoted = SExtPromotedInteger(Op1);
-    Op2Promoted = SExtPromotedInteger(Op2);
+    Op1 = SExtPromotedInteger(Op1);
+    Op2 = SExtPromotedInteger(Op2);
   }
-  EVT PromotedType = Op1Promoted.getValueType();
+  EVT PromotedType = Op1.getValueType();
   unsigned NewBits = PromotedType.getScalarSizeInBits();
 
-  if (Opcode == ISD::UADDSAT) {
-    APInt MaxVal = APInt::getLowBitsSet(NewBits, OldBits);
-    SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
-    SDValue Add =
-        matcher.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
-    return matcher.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
-  }
-
   // Shift cannot use a min/max expansion, we can't detect overflow if all of
   // the bits have been shifted out.
   if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
@@ -1102,14 +1109,11 @@ SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
     unsigned SHLAmount = NewBits - OldBits;
     SDValue ShiftAmount =
         DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
-    Op1Promoted =
-        DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
+    Op1 = DAG.getNode(ISD::SHL, dl, PromotedType, Op1, ShiftAmount);
     if (!IsShift)
-      Op2Promoted =
-          matcher.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
+      Op2 = matcher.getNode(ISD::SHL, dl, PromotedType, Op2, ShiftAmount);
 
-    SDValue Result =
-        matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
+    SDValue Result = matcher.getNode(Opcode, dl, PromotedType, Op1, Op2);
     return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
   }
 
@@ -1118,8 +1122,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
   APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
   SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
   SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
-  SDValue Result =
-      matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
+  SDValue Result = matcher.getNode(AddOp, dl, PromotedType, Op1, Op2);
   Result = matcher.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
   Result = matcher.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
   return Result;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index d4979024cb57bdc..b90addbfba04af3 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -59,7 +59,6 @@ static std::vector<Desc> getOpDescriptions() {
   Descriptions[DW_OP_shr] = Desc(Op::Dwarf2);
   Descriptions[DW_OP_shra] = Desc(Op::Dwarf2);
   Descriptions[DW_OP_xor] = Desc(Op::Dwarf2);
-  Descriptions[DW_OP_skip] = Desc(Op::Dwarf2, Op::SignedSize2);
   Descriptions[DW_OP_bra] = Desc(Op::Dwarf2, Op::SignedSize2);
   Descriptions[DW_OP_eq] = Desc(Op::Dwarf2);
   Descriptions[DW_OP_ge] = Desc(Op::Dwarf2);
@@ -67,6 +66,7 @@ static std::vector<Desc> getOpDescriptions() {
   Descriptions[DW_OP_le] = Desc(Op::Dwarf2);
   Descriptions[DW_OP_lt] = Desc(Op::Dwarf2);
   Descriptions[DW_OP_ne] = Desc(Op::Dwarf2);
+  Descriptions[DW_OP_skip] = Desc(Op::Dwarf2, Op::SignedSize2);
   for (uint16_t LA = DW_OP_lit0; LA <= DW_OP_lit31; ++LA)
     Descriptions[LA] = Desc(Op::Dwarf2);
   for (uint16_t LA = DW_OP_reg0; LA <= DW_OP_reg31; ++LA)
@@ -88,20 +88,20 @@ static std::vector<Desc> getOpDescriptions() {
   Descriptions[DW_OP_call_frame_cfa] = Desc(Op::Dwarf3);
   Descriptions[DW_OP_bit_piece] = Desc(Op::Dwarf3, Op::SizeLEB, Op::SizeLEB);
   Descriptions[DW_OP_implicit_value] =
-      Desc(Op::Dwarf3, Op::SizeLEB, Op::SizeBlock);
-  Descriptions[DW_OP_stack_value] = Desc(Op::Dwarf3);
+      Desc(Op::Dwarf4, Op::SizeLEB, Op::SizeBlock);
+  Descriptions[DW_OP_stack_value] = Desc(Op::Dwarf4);
+  Descriptions[DW_OP_addrx] = Desc(Op::Dwarf5, Op::SizeLEB);
+  Descriptions[DW_OP_constx] = Desc(Op::Dwarf5, Op::SizeLEB);
+  Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB);
+  Descriptions[DW_OP_convert] = Desc(Op::Dwarf5, Op::BaseTypeRef);
+  Descriptions[DW_OP_regval_type] =
+      Desc(Op::Dwarf5, Op::SizeLEB, Op::BaseTypeRef);
   Descriptions[DW_OP_WASM_location] =
       Desc(Op::Dwarf4, Op::SizeLEB, Op::WasmLocationArg);
   Descriptions[DW_OP_GNU_push_tls_address] = Desc(Op::Dwarf3);
   Descriptions[DW_OP_GNU_addr_index] = Desc(Op::Dwarf4, Op::SizeLEB);
   Descriptions[DW_OP_GNU_const_index] = Desc(Op::Dwarf4, Op::SizeLEB);
   Descriptions[DW_OP_GNU_entry_value] = Desc(Op::Dwarf4, Op::SizeLEB);
-  Descriptions[DW_OP_addrx] = Desc(Op::Dwarf5, Op::SizeLEB);
-  Descriptions[DW_OP_constx] = Desc(Op::Dwarf5, Op::SizeLEB);
-  Descriptions[DW_OP_convert] = Desc(Op::Dwarf5, Op::BaseTypeRef);
-  Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB);
-  Descriptions[DW_OP_regval_type] =
-      Desc(Op::Dwarf5, Op::SizeLEB, Op::BaseTypeRef);
   // This Description acts as a marker that getSubOpDesc must be called
   // to fetch the final Description for the operation. Each such final
   // Description must share the same first SizeSubOpLEB operand.
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index b481520fa6c6f9b..83fec194d739045 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5907,7 +5907,7 @@ CallInst *OpenMPIRBuilder::createOMPInteropInit(
   Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
   Value *ThreadId = getOrCreateThreadID(Ident);
   if (Device == nullptr)
-    Device = ConstantInt::get(Int32, -1);
+    Device = Constant::getAllOnesValue(Int32);
   Constant *InteropTypeVal = ConstantInt::get(Int32, (int)InteropType);
   if (NumDependences == nullptr) {
     NumDependences = ConstantInt::get(Int32, 0);
@@ -5935,7 +5935,7 @@ CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
   Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
   Value *ThreadId = getOrCreateThreadID(Ident);
   if (Device == nullptr)
-    Device = ConstantInt::get(Int32, -1);
+    Device = Constant::getAllOnesValue(Int32);
   if (NumDependences == nullptr) {
     NumDependences = ConstantInt::get(Int32, 0);
     PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());
@@ -5963,7 +5963,7 @@ CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
   Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
   Value *ThreadId = getOrCreateThreadID(Ident);
   if (Device == nullptr)
-    Device = ConstantInt::get(Int32, -1);
+    Device = Constant::getAllOnesValue(Int32);
   if (NumDependences == nullptr) {
     NumDependences = ConstantInt::get(Int32, 0);
     PointerType *PointerTypeVar = PointerType::getUnqual(M.getContext());
@@ -6104,7 +6104,7 @@ OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
       Builder.CreateCall(Fn, {KernelEnvironment, KernelLaunchEnvironment});
 
   Value *ExecUserCode = Builder.CreateICmpEQ(
-      ThreadKind, ConstantInt::get(ThreadKind->getType(), -1),
+      ThreadKind, Constant::getAllOnesValue(ThreadKind->getType()),
       "exec_user_code");
 
   // ThreadKind = __kmpc_target_init(...)
diff --git a/llvm/lib/Object/Minidump.cpp b/llvm/lib/Object/Minidump.cpp
index 6febff89ac51956..93b2e2cecd10531 100644
--- a/llvm/lib/Object/Minidump.cpp
+++ b/llvm/lib/Object/Minidump.cpp
@@ -99,8 +99,9 @@ template Expected<ArrayRef<Thread>>
 template Expected<ArrayRef<MemoryDescriptor>>
     MinidumpFile::getListStream(StreamType) const;
 
-Expected<ArrayRef<uint8_t>>
-MinidumpFile::getDataSlice(ArrayRef<uint8_t> Data, size_t Offset, size_t Size) {
+Expected<ArrayRef<uint8_t>> MinidumpFile::getDataSlice(ArrayRef<uint8_t> Data,
+                                                       uint64_t Offset,
+                                                       uint64_t Size) {
   // Check for overflow.
   if (Offset + Size < Offset || Offset + Size < Size ||
       Offset + Size > Data.size())
@@ -154,3 +155,43 @@ MinidumpFile::create(MemoryBufferRef Source) {
   return std::unique_ptr<MinidumpFile>(
       new MinidumpFile(Source, Hdr, *ExpectedStreams, std::move(StreamMap)));
 }
+
+iterator_range<MinidumpFile::FallibleMemory64Iterator>
+MinidumpFile::getMemory64List(Error &Err) const {
+  ErrorAsOutParameter ErrAsOutParam(&Err);
+  auto end = FallibleMemory64Iterator::end(Memory64Iterator::end());
+  Expected<minidump::Memory64ListHeader> ListHeader = getMemoryList64Header();
+  if (!ListHeader) {
+    Err = ListHeader.takeError();
+    return make_range(end, end);
+  }
+
+  std::optional<ArrayRef<uint8_t>> Stream =
+      getRawStream(StreamType::Memory64List);
+  if (!Stream) {
+    Err = createError("No such stream");
+    return make_range(end, end);
+  }
+
+  Expected<ArrayRef<minidump::MemoryDescriptor_64>> Descriptors =
+      getDataSliceAs<minidump::MemoryDescriptor_64>(
+          *Stream, sizeof(Memory64ListHeader),
+          ListHeader->NumberOfMemoryRanges);
+
+  if (!Descriptors) {
+    Err = Descriptors.takeError();
+    return make_range(end, end);
+  }
+
+  if (!Descriptors->empty() &&
+      ListHeader->BaseRVA + Descriptors->front().DataSize > getData().size()) {
+    Err = createError("Memory64List header RVA out of range");
+    return make_range(end, end);
+  }
+
+  return make_range(FallibleMemory64Iterator::itr(
+                        Memory64Iterator::begin(
+                            getData().slice(ListHeader->BaseRVA), *Descriptors),
+                        Err),
+                    FallibleMemory64Iterator::end(Memory64Iterator::end()));
+}
diff --git a/llvm/lib/ObjectYAML/MinidumpEmitter.cpp b/llvm/lib/ObjectYAML/MinidumpEmitter.cpp
index 24b521a9925c73c..44cdfbdd80ea537 100644
--- a/llvm/lib/ObjectYAML/MinidumpEmitter.cpp
+++ b/llvm/lib/ObjectYAML/MinidumpEmitter.cpp
@@ -136,6 +136,30 @@ static size_t layout(BlobAllocator &File, MinidumpYAML::ExceptionStream &S) {
   return DataEnd;
 }
 
+static size_t layout(BlobAllocator &File, MinidumpYAML::Memory64ListStream &S) {
+  size_t BaseRVA = File.tell() + sizeof(minidump::Memory64ListHeader);
+  BaseRVA += S.Entries.size() * sizeof(minidump::MemoryDescriptor_64);
+  S.Header.BaseRVA = BaseRVA;
+  S.Header.NumberOfMemoryRanges = S.Entries.size();
+  File.allocateObject(S.Header);
+  for (auto &E : S.Entries)
+    File.allocateObject(E.Entry);
+
+  // Save the new offset for the stream size.
+  size_t DataEnd = File.tell();
+  for (auto &E : S.Entries) {
+    File.allocateBytes(E.Content);
+    if (E.Entry.DataSize > E.Content.binary_size()) {
+      size_t Padding = E.Entry.DataSize - E.Content.binary_size();
+      File.allocateCallback(Padding, [Padding](raw_ostream &OS) {
+        OS << std::string(Padding, '\0');
+      });
+    }
+  }
+
+  return DataEnd;
+}
+
 static void layout(BlobAllocator &File, MemoryListStream::entry_type &Range) {
   Range.Entry.Memory = layout(File, Range.Content);
 }
@@ -190,6 +214,9 @@ static Directory layout(BlobAllocator &File, Stream &S) {
   case Stream::StreamKind::MemoryList:
     DataEnd = layout(File, cast<MemoryListStream>(S));
     break;
+  case Stream::StreamKind::Memory64List:
+    DataEnd = layout(File, cast<Memory64ListStream>(S));
+    break;
   case Stream::StreamKind::ModuleList:
     DataEnd = layout(File, cast<ModuleListStream>(S));
     break;
diff --git a/llvm/lib/ObjectYAML/MinidumpYAML.cpp b/llvm/lib/ObjectYAML/MinidumpYAML.cpp
index fdbd2d8e6dcbc75..10b8676b5c4cfb9 100644
--- a/llvm/lib/ObjectYAML/MinidumpYAML.cpp
+++ b/llvm/lib/ObjectYAML/MinidumpYAML.cpp
@@ -75,6 +75,8 @@ Stream::StreamKind Stream::getKind(StreamType Type) {
     return StreamKind::MemoryInfoList;
   case StreamType::MemoryList:
     return StreamKind::MemoryList;
+  case StreamType::Memory64List:
+    return StreamKind::Memory64List;
   case StreamType::ModuleList:
     return StreamKind::ModuleList;
   case StreamType::SystemInfo:
@@ -103,6 +105,8 @@ std::unique_ptr<Stream> Stream::create(StreamType Type) {
     return std::make_unique<MemoryInfoListStream>();
   case StreamKind::MemoryList:
     return std::make_unique<MemoryListStream>();
+  case StreamKind::Memory64List:
+    return std::make_unique<Memory64ListStream>();
   case StreamKind::ModuleList:
     return std::make_unique<ModuleListStream>();
   case StreamKind::RawContent:
@@ -256,6 +260,12 @@ void yaml::MappingTraits<MemoryInfo>::mapping(IO &IO, MemoryInfo &Info) {
   mapOptionalHex(IO, "Reserved1", Info.Reserved1, 0);
 }
 
+void yaml::MappingTraits<Memory64ListStream::entry_type>::mapping(
+    IO &IO, Memory64ListStream::entry_type &Mem) {
+  MappingContextTraits<MemoryDescriptor_64, yaml::BinaryRef>::mapping(
+      IO, Mem.Entry, Mem.Content);
+}
+
 void yaml::MappingTraits<VSFixedFileInfo>::mapping(IO &IO,
                                                    VSFixedFileInfo &Info) {
   mapOptionalHex(IO, "Signature", Info.Signature, 0);
@@ -312,6 +322,18 @@ static void streamMapping(yaml::IO &IO, MemoryListStream &Stream) {
   IO.mapRequired("Memory Ranges", Stream.Entries);
 }
 
+static void streamMapping(yaml::IO &IO, Memory64ListStream &Stream) {
+  IO.mapRequired("Memory Ranges", Stream.Entries);
+}
+
+static std::string streamValidate(Memory64ListStream &Stream) {
+  for (auto &Entry : Stream.Entries) {
+    if (Entry.Entry.DataSize < Entry.Content.binary_size())
+      return "Memory region size must be greater or equal to the content size";
+  }
+  return "";
+}
+
 static void streamMapping(yaml::IO &IO, ModuleListStream &Stream) {
   IO.mapRequired("Modules", Stream.Entries);
 }
@@ -356,6 +378,13 @@ void yaml::MappingContextTraits<MemoryDescriptor, yaml::BinaryRef>::mapping(
   IO.mapRequired("Content", Content);
 }
 
+void yaml::MappingContextTraits<MemoryDescriptor_64, yaml::BinaryRef>::mapping(
+    IO &IO, MemoryDescriptor_64 &Memory, BinaryRef &Content) {
+  mapRequiredHex(IO, "Start of Memory Range", Memory.StartOfMemoryRange);
+  IO.mapRequired("Content", Content);
+  mapOptional(IO, "Data Size", Memory.DataSize, Content.binary_size());
+}
+
 void yaml::MappingTraits<ThreadListStream::entry_type>::mapping(
     IO &IO, ThreadListStream::entry_type &T) {
   mapRequiredHex(IO, "Thread Id", T.Entry.ThreadId);
@@ -416,6 +445,9 @@ void yaml::MappingTraits<std::unique_ptr<Stream>>::mapping(
   case MinidumpYAML::Stream::StreamKind::MemoryList:
     streamMapping(IO, llvm::cast<MemoryListStream>(*S));
     break;
+  case MinidumpYAML::Stream::StreamKind::Memory64List:
+    streamMapping(IO, llvm::cast<Memory64ListStream>(*S));
+    break;
   case MinidumpYAML::Stream::StreamKind::ModuleList:
     streamMapping(IO, llvm::cast<ModuleListStream>(*S));
     break;
@@ -439,6 +471,8 @@ std::string yaml::MappingTraits<std::unique_ptr<Stream>>::validate(
   switch (S->Kind) {
   case MinidumpYAML::Stream::StreamKind::RawContent:
     return streamValidate(cast<RawContentStream>(*S));
+  case MinidumpYAML::Stream::StreamKind::Memory64List:
+    return streamValidate(cast<Memory64ListStream>(*S));
   case MinidumpYAML::Stream::StreamKind::Exception:
   case MinidumpYAML::Stream::StreamKind::MemoryInfoList:
   case MinidumpYAML::Stream::StreamKind::MemoryList:
@@ -494,6 +528,18 @@ Stream::create(const Directory &StreamDesc, const object::MinidumpFile &File) {
     }
     return std::make_unique<MemoryListStream>(std::move(Ranges));
   }
+  case StreamKind::Memory64List: {
+    Error Err = Error::success();
+    auto Memory64List = File.getMemory64List(Err);
+    std::vector<Memory64ListStream::entry_type> Ranges;
+    for (const auto &Pair : Memory64List) {
+      Ranges.push_back({Pair.first, Pair.second});
+    }
+
+    if (Err)
+      return Err;
+    return std::make_unique<Memory64ListStream>(std::move(Ranges));
+  }
   case StreamKind::ModuleList: {
     auto ExpectedList = File.getModuleList();
     if (!ExpectedList)
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 46f43f3de4705c3..1859fde279c98dd 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Analysis/CycleAnalysis.h"
 #include "llvm/Analysis/DDG.h"
 #include "llvm/Analysis/DDGPrinter.h"
+#include "llvm/Analysis/DXILMetadataAnalysis.h"
 #include "llvm/Analysis/Delinearization.h"
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 0cec9fbd7cd05e4..5ef8ba309444706 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -21,6 +21,7 @@
 MODULE_ANALYSIS("callgraph", CallGraphAnalysis())
 MODULE_ANALYSIS("collector-metadata", CollectorMetadataAnalysis())
 MODULE_ANALYSIS("ctx-prof-analysis", CtxProfAnalysis(UseCtxProfile))
+MODULE_ANALYSIS("dxil-metadata", DXILMetadataAnalysis())
 MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())
 MODULE_ANALYSIS("ir-similarity", IRSimilarityAnalysis())
 MODULE_ANALYSIS("lcg", LazyCallGraphAnalysis())
@@ -117,6 +118,7 @@ MODULE_PASS("print-must-be-executed-contexts",
             MustBeExecutedContextPrinterPass(dbgs()))
 MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs()))
 MODULE_PASS("print-stack-safety", StackSafetyGlobalPrinterPass(dbgs()))
+MODULE_PASS("print<dxil-metadata>", DXILMetadataAnalysisPrinterPass(dbgs()))
 MODULE_PASS("print<inline-advisor>", InlineAdvisorAnalysisPrinterPass(dbgs()))
 MODULE_PASS("print<module-debuginfo>", ModuleDebugInfoPrinterPass(dbgs()))
 MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM))
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 192c53d1a13f19a..445f56b14e83b5b 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1235,6 +1235,30 @@ Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
   return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
 }
 
+Value *ExtractElementInst::create(Value *Vec, Value *Idx,
+                                  Instruction *InsertBefore, Context &Ctx,
+                                  const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+  llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
+  if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
+    return Ctx.createExtractElementInst(NewExtract);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *ExtractElementInst::create(Value *Vec, Value *Idx,
+                                  BasicBlock *InsertAtEnd, Context &Ctx,
+                                  const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
+  if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
+    return Ctx.createExtractElementInst(NewExtract);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
 Constant *Constant::createInt(Type *Ty, uint64_t V, Context &Ctx,
                               bool IsSigned) {
   llvm::Constant *LLVMC = llvm::ConstantInt::get(Ty, V, IsSigned);
@@ -1356,6 +1380,12 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
     It->second = std::unique_ptr<SelectInst>(new SelectInst(LLVMSel, *this));
     return It->second.get();
   }
+  case llvm::Instruction::ExtractElement: {
+    auto *LLVMIns = cast<llvm::ExtractElementInst>(LLVMV);
+    It->second = std::unique_ptr<ExtractElementInst>(
+        new ExtractElementInst(LLVMIns, *this));
+    return It->second.get();
+  }
   case llvm::Instruction::InsertElement: {
     auto *LLVMIns = cast<llvm::InsertElementInst>(LLVMV);
     It->second = std::unique_ptr<InsertElementInst>(
@@ -1459,6 +1489,13 @@ SelectInst *Context::createSelectInst(llvm::SelectInst *SI) {
   return cast<SelectInst>(registerValue(std::move(NewPtr)));
 }
 
+ExtractElementInst *
+Context::createExtractElementInst(llvm::ExtractElementInst *EEI) {
+  auto NewPtr =
+      std::unique_ptr<ExtractElementInst>(new ExtractElementInst(EEI, *this));
+  return cast<ExtractElementInst>(registerValue(std::move(NewPtr)));
+}
+
 InsertElementInst *
 Context::createInsertElementInst(llvm::InsertElementInst *IEI) {
   auto NewPtr =
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7777aa4b50a3707..efc1703221d21e7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -17202,17 +17202,148 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
   return true;
 }
 
+bool getDeinterleave2Values(
+    Value *DI, SmallVectorImpl<Instruction *> &DeinterleavedValues,
+    SmallVectorImpl<Instruction *> &DeInterleaveDeadInsts) {
+  if (!DI->hasNUses(2))
+    return false;
+  auto *Extr1 = dyn_cast<ExtractValueInst>(*(DI->user_begin()));
+  auto *Extr2 = dyn_cast<ExtractValueInst>(*(++DI->user_begin()));
+  if (!Extr1 || !Extr2)
+    return false;
+
+  DeinterleavedValues.resize(2);
+  // Place the values into the vector in the order of extraction:
+  DeinterleavedValues[0x1 & (Extr1->getIndices()[0])] = Extr1;
+  DeinterleavedValues[0x1 & (Extr2->getIndices()[0])] = Extr2;
+  if (!DeinterleavedValues[0] || !DeinterleavedValues[1])
+    return false;
+
+  // Make sure that the extracted values match the deinterleave tree pattern
+  if (!match(DeinterleavedValues[0], m_ExtractValue<0>((m_Specific(DI)))) ||
+      !match(DeinterleavedValues[1], m_ExtractValue<1>((m_Specific(DI))))) {
+    LLVM_DEBUG(dbgs() << "matching deinterleave2 failed\n");
+    return false;
+  }
+  // DeinterleavedValues will be replace by output of ld2
+  DeInterleaveDeadInsts.insert(DeInterleaveDeadInsts.end(),
+                               DeinterleavedValues.begin(),
+                               DeinterleavedValues.end());
+  return true;
+}
+
+/*
+DeinterleaveIntrinsic tree:
+                   [DI]
+                /        \
+         [Extr<0>]      [Extr<1>]
+            |                 |
+           [DI]              [DI]
+          /    \            /    \
+    [Extr<0>][Extr<1>] [Extr<0>][Extr<1>]
+        |       |         |         |
+roots:  A       C         B         D
+roots in correct order of DI4 will be: A B C D.
+Returns true if `DI` is the top of an IR tree that represents a theoretical
+vector.deinterleave4 intrinsic. When true is returned, \p `DeinterleavedValues`
+vector is populated with the results such an intrinsic would return: (i.e. {A,
+B, C, D } = vector.deinterleave4(...))
+*/
+bool getDeinterleave4Values(
+    Value *DI, SmallVectorImpl<Instruction *> &DeinterleavedValues,
+    SmallVectorImpl<Instruction *> &DeInterleaveDeadInsts) {
+  if (!DI->hasNUses(2))
+    return false;
+  auto *Extr1 = dyn_cast<ExtractValueInst>(*(DI->user_begin()));
+  auto *Extr2 = dyn_cast<ExtractValueInst>(*(++DI->user_begin()));
+  if (!Extr1 || !Extr2)
+    return false;
+
+  if (!Extr1->hasOneUse() || !Extr2->hasOneUse())
+    return false;
+  auto *DI1 = *(Extr1->user_begin());
+  auto *DI2 = *(Extr2->user_begin());
+
+  if (!DI1->hasNUses(2) || !DI2->hasNUses(2))
+    return false;
+  // Leaf nodes of the deinterleave tree:
+  auto *A = dyn_cast<ExtractValueInst>(*(DI1->user_begin()));
+  auto *C = dyn_cast<ExtractValueInst>(*(++DI1->user_begin()));
+  auto *B = dyn_cast<ExtractValueInst>(*(DI2->user_begin()));
+  auto *D = dyn_cast<ExtractValueInst>(*(++DI2->user_begin()));
+  // Make sure that the A,B,C and D are ExtractValue instructions before getting
+  // the extract index
+  if (!A || !B || !C || !D)
+    return false;
+
+  DeinterleavedValues.resize(4);
+  // Place the values into the vector in the order of deinterleave4:
+  DeinterleavedValues[0x3 &
+                      ((A->getIndices()[0] * 2) + Extr1->getIndices()[0])] = A;
+  DeinterleavedValues[0x3 &
+                      ((B->getIndices()[0] * 2) + Extr2->getIndices()[0])] = B;
+  DeinterleavedValues[0x3 &
+                      ((C->getIndices()[0] * 2) + Extr1->getIndices()[0])] = C;
+  DeinterleavedValues[0x3 &
+                      ((D->getIndices()[0] * 2) + Extr2->getIndices()[0])] = D;
+  if (!DeinterleavedValues[0] || !DeinterleavedValues[1] ||
+      !DeinterleavedValues[2] || !DeinterleavedValues[3])
+    return false;
+
+  // Make sure that A,B,C,D match the deinterleave tree pattern
+  if (!match(DeinterleavedValues[0], m_ExtractValue<0>(m_Deinterleave2(
+                                         m_ExtractValue<0>(m_Specific(DI))))) ||
+      !match(DeinterleavedValues[1], m_ExtractValue<0>(m_Deinterleave2(
+                                         m_ExtractValue<1>(m_Specific(DI))))) ||
+      !match(DeinterleavedValues[2], m_ExtractValue<1>(m_Deinterleave2(
+                                         m_ExtractValue<0>(m_Specific(DI))))) ||
+      !match(DeinterleavedValues[3], m_ExtractValue<1>(m_Deinterleave2(
+                                         m_ExtractValue<1>(m_Specific(DI)))))) {
+    LLVM_DEBUG(dbgs() << "matching deinterleave4 failed\n");
+    return false;
+  }
+
+  // These Values will not be used anymore,
+  // DI4 will be created instead of nested DI1 and DI2
+  DeInterleaveDeadInsts.insert(DeInterleaveDeadInsts.end(),
+                               DeinterleavedValues.begin(),
+                               DeinterleavedValues.end());
+  DeInterleaveDeadInsts.push_back(cast<Instruction>(DI1));
+  DeInterleaveDeadInsts.push_back(cast<Instruction>(Extr1));
+  DeInterleaveDeadInsts.push_back(cast<Instruction>(DI2));
+  DeInterleaveDeadInsts.push_back(cast<Instruction>(Extr2));
+
+  return true;
+}
+
+bool getDeinterleavedValues(
+    Value *DI, SmallVectorImpl<Instruction *> &DeinterleavedValues,
+    SmallVectorImpl<Instruction *> &DeInterleaveDeadInsts) {
+  if (getDeinterleave4Values(DI, DeinterleavedValues, DeInterleaveDeadInsts))
+    return true;
+  return getDeinterleave2Values(DI, DeinterleavedValues, DeInterleaveDeadInsts);
+}
+
 bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
-    IntrinsicInst *DI, LoadInst *LI) const {
+    IntrinsicInst *DI, LoadInst *LI,
+    SmallVectorImpl<Instruction *> &DeadInsts) const {
   // Only deinterleave2 supported at present.
   if (DI->getIntrinsicID() != Intrinsic::vector_deinterleave2)
     return false;
 
-  // Only a factor of 2 supported at present.
-  const unsigned Factor = 2;
+  SmallVector<Instruction *, 4> DeinterleavedValues;
+  SmallVector<Instruction *, 8> DeInterleaveDeadInsts;
 
-  VectorType *VTy = cast<VectorType>(DI->getType()->getContainedType(0));
-  const DataLayout &DL = DI->getDataLayout();
+  if (!getDeinterleavedValues(DI, DeinterleavedValues, DeInterleaveDeadInsts)) {
+    LLVM_DEBUG(dbgs() << "Matching ld2 and ld4 patterns failed\n");
+    return false;
+  }
+  unsigned Factor = DeinterleavedValues.size();
+  assert((Factor == 2 || Factor == 4) &&
+         "Currently supported Factor is 2 or 4 only");
+  VectorType *VTy = cast<VectorType>(DeinterleavedValues[0]->getType());
+
+  const DataLayout &DL = DI->getModule()->getDataLayout();
   bool UseScalable;
   if (!isLegalInterleavedAccessType(VTy, DL, UseScalable))
     return false;
@@ -17223,7 +17354,6 @@ bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
     return false;
 
   unsigned NumLoads = getNumInterleavedAccesses(VTy, DL, UseScalable);
-
   VectorType *LdTy =
       VectorType::get(VTy->getElementType(),
                       VTy->getElementCount().divideCoefficientBy(NumLoads));
@@ -17233,18 +17363,15 @@ bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
                                                 UseScalable, LdTy, PtrTy);
 
   IRBuilder<> Builder(LI);
-
   Value *Pred = nullptr;
   if (UseScalable)
     Pred =
         Builder.CreateVectorSplat(LdTy->getElementCount(), Builder.getTrue());
 
   Value *BaseAddr = LI->getPointerOperand();
-  Value *Result;
   if (NumLoads > 1) {
-    Value *Left = PoisonValue::get(VTy);
-    Value *Right = PoisonValue::get(VTy);
-
+    // Create multiple legal small ldN.
+    SmallVector<Value *, 4> ExtractedLdValues(Factor, PoisonValue::get(VTy));
     for (unsigned I = 0; I < NumLoads; ++I) {
       Value *Offset = Builder.getInt64(I * Factor);
 
@@ -17254,40 +17381,96 @@ bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
         LdN = Builder.CreateCall(LdNFunc, {Pred, Address}, "ldN");
       else
         LdN = Builder.CreateCall(LdNFunc, Address, "ldN");
-
       Value *Idx =
           Builder.getInt64(I * LdTy->getElementCount().getKnownMinValue());
-      Left = Builder.CreateInsertVector(
-          VTy, Left, Builder.CreateExtractValue(LdN, 0), Idx);
-      Right = Builder.CreateInsertVector(
-          VTy, Right, Builder.CreateExtractValue(LdN, 1), Idx);
+      for (unsigned J = 0; J < Factor; ++J) {
+        ExtractedLdValues[J] = Builder.CreateInsertVector(
+            VTy, ExtractedLdValues[J], Builder.CreateExtractValue(LdN, J), Idx);
+      }
+      LLVM_DEBUG(dbgs() << "LdN4 res: "; LdN->dump());
     }
-
-    Result = PoisonValue::get(DI->getType());
-    Result = Builder.CreateInsertValue(Result, Left, 0);
-    Result = Builder.CreateInsertValue(Result, Right, 1);
+    // Replace output of deinterleave2 intrinsic by output of ldN2/ldN4
+    for (unsigned J = 0; J < Factor; ++J)
+      DeinterleavedValues[J]->replaceAllUsesWith(ExtractedLdValues[J]);
   } else {
+    Value *Result;
     if (UseScalable)
       Result = Builder.CreateCall(LdNFunc, {Pred, BaseAddr}, "ldN");
     else
       Result = Builder.CreateCall(LdNFunc, BaseAddr, "ldN");
+    // Replace output of deinterleave2 intrinsic by output of ldN2/ldN4
+    for (unsigned I = 0; I < Factor; I++) {
+      Value *NewExtract = Builder.CreateExtractValue(Result, I);
+      DeinterleavedValues[I]->replaceAllUsesWith(NewExtract);
+    }
   }
-
-  DI->replaceAllUsesWith(Result);
+  DeadInsts.insert(DeadInsts.end(), DeInterleaveDeadInsts.begin(),
+                   DeInterleaveDeadInsts.end());
   return true;
 }
 
+/*
+InterleaveIntrinsic tree.
+          A    C         B    D
+           \  /           \  /
+           [II]           [II]
+                 \     /
+                  [II]
+
+values in correct order of interleave4: A B C D.
+Returns true if `II` is the root of an IR tree that represents a theoretical
+vector.interleave4 intrinsic. When true is returned, \p `InterleavedValues`
+vector is populated with the inputs such an intrinsic would take: (i.e.
+vector.interleave4(A, B, C, D)).
+*/
+bool getValuesToInterleave(
+    Value *II, SmallVectorImpl<Value *> &InterleavedValues,
+    SmallVectorImpl<Instruction *> &InterleaveDeadInsts) {
+  Value *A, *B, *C, *D;
+  // Try to match interleave of Factor 4
+  if (match(II, m_Interleave2(m_Interleave2(m_Value(A), m_Value(C)),
+                              m_Interleave2(m_Value(B), m_Value(D))))) {
+    InterleavedValues.push_back(A);
+    InterleavedValues.push_back(B);
+    InterleavedValues.push_back(C);
+    InterleavedValues.push_back(D);
+    // intermediate II will not be needed anymore
+    InterleaveDeadInsts.push_back(
+        cast<Instruction>(cast<Instruction>(II)->getOperand(0)));
+    InterleaveDeadInsts.push_back(
+        cast<Instruction>(cast<Instruction>(II)->getOperand(1)));
+    return true;
+  }
+
+  // Try to match interleave of Factor 2
+  if (match(II, m_Interleave2(m_Value(A), m_Value(B)))) {
+    InterleavedValues.push_back(A);
+    InterleavedValues.push_back(B);
+    return true;
+  }
+
+  return false;
+}
+
 bool AArch64TargetLowering::lowerInterleaveIntrinsicToStore(
-    IntrinsicInst *II, StoreInst *SI) const {
+    IntrinsicInst *II, StoreInst *SI,
+    SmallVectorImpl<Instruction *> &DeadInsts) const {
   // Only interleave2 supported at present.
   if (II->getIntrinsicID() != Intrinsic::vector_interleave2)
     return false;
 
-  // Only a factor of 2 supported at present.
-  const unsigned Factor = 2;
+  SmallVector<Value *, 4> InterleavedValues;
+  SmallVector<Instruction *, 2> InterleaveDeadInsts;
+  if (!getValuesToInterleave(II, InterleavedValues, InterleaveDeadInsts)) {
+    LLVM_DEBUG(dbgs() << "Matching st2 and st4 patterns failed\n");
+    return false;
+  }
+  unsigned Factor = InterleavedValues.size();
+  assert((Factor == 2 || Factor == 4) &&
+         "Currently supported Factor is 2 or 4 only");
+  VectorType *VTy = cast<VectorType>(InterleavedValues[0]->getType());
+  const DataLayout &DL = II->getModule()->getDataLayout();
 
-  VectorType *VTy = cast<VectorType>(II->getOperand(0)->getType());
-  const DataLayout &DL = II->getDataLayout();
   bool UseScalable;
   if (!isLegalInterleavedAccessType(VTy, DL, UseScalable))
     return false;
@@ -17316,27 +17499,28 @@ bool AArch64TargetLowering::lowerInterleaveIntrinsicToStore(
     Pred =
         Builder.CreateVectorSplat(StTy->getElementCount(), Builder.getTrue());
 
-  Value *L = II->getOperand(0);
-  Value *R = II->getOperand(1);
-
+  auto ExtractedValues = InterleavedValues;
+  if (UseScalable)
+    InterleavedValues.push_back(Pred);
+  InterleavedValues.push_back(BaseAddr);
   for (unsigned I = 0; I < NumStores; ++I) {
     Value *Address = BaseAddr;
     if (NumStores > 1) {
       Value *Offset = Builder.getInt64(I * Factor);
       Address = Builder.CreateGEP(StTy, BaseAddr, {Offset});
-
       Value *Idx =
           Builder.getInt64(I * StTy->getElementCount().getKnownMinValue());
-      L = Builder.CreateExtractVector(StTy, II->getOperand(0), Idx);
-      R = Builder.CreateExtractVector(StTy, II->getOperand(1), Idx);
+      for (unsigned J = 0; J < Factor; J++) {
+        InterleavedValues[J] =
+            Builder.CreateExtractVector(StTy, ExtractedValues[J], Idx);
+      }
+      // update the address
+      InterleavedValues[InterleavedValues.size() - 1] = Address;
     }
-
-    if (UseScalable)
-      Builder.CreateCall(StNFunc, {L, R, Pred, Address});
-    else
-      Builder.CreateCall(StNFunc, {L, R, Address});
+    Builder.CreateCall(StNFunc, InterleavedValues);
   }
-
+  DeadInsts.insert(DeadInsts.end(), InterleaveDeadInsts.begin(),
+                   InterleaveDeadInsts.end());
   return true;
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 81e15185f985d50..ace682fb89379d2 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -703,11 +703,13 @@ class AArch64TargetLowering : public TargetLowering {
   bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
                              unsigned Factor) const override;
 
-  bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *DI,
-                                        LoadInst *LI) const override;
+  bool lowerDeinterleaveIntrinsicToLoad(
+      IntrinsicInst *DI, LoadInst *LI,
+      SmallVectorImpl<Instruction *> &DeadInsts) const override;
 
-  bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II,
-                                       StoreInst *SI) const override;
+  bool lowerInterleaveIntrinsicToStore(
+      IntrinsicInst *II, StoreInst *SI,
+      SmallVectorImpl<Instruction *> &DeadInsts) const override;
 
   bool isLegalAddImmediate(int64_t) const override;
   bool isLegalAddScalableImmediate(int64_t) const override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 1ddf6686b97e74f..1ad2302a4f20c62 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -1430,19 +1430,16 @@ bool MFMAExpInterleaveOpt::analyzeDAG(const SIInstrInfo *TII) {
   if (!(TempExp && TempMFMA))
     return false;
 
-  HasChainBetweenCvt =
-      std::find_if((*TempExp)->Succs.begin(), (*TempExp)->Succs.end(),
-                   [&isCvt](SDep &Succ) {
-                     return isCvt(Succ.getSUnit()->getInstr()->getOpcode());
-                   }) == (*TempExp)->Succs.end();
+  HasChainBetweenCvt = none_of((*TempExp)->Succs, [&isCvt](SDep &Succ) {
+    return isCvt(Succ.getSUnit()->getInstr()->getOpcode());
+  });
 
   // Count the number of MFMAs that are reached by an EXP
   for (auto &SuccSU : MFMAPipeCands) {
     if (MFMAPipeSUs.size() &&
-        std::find_if(MFMAPipeSUs.begin(), MFMAPipeSUs.end(),
-                     [&SuccSU](SUnit *PotentialMatch) {
-                       return PotentialMatch->NodeNum == SuccSU->NodeNum;
-                     }) != MFMAPipeSUs.end())
+        any_of(MFMAPipeSUs, [&SuccSU](SUnit *PotentialMatch) {
+          return PotentialMatch->NodeNum == SuccSU->NodeNum;
+        }))
       continue;
 
     for (auto &PredSU : ExpPipeCands) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 423fa91a690684d..05906a1335895d1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -383,6 +383,11 @@ static cl::opt<bool> EnableHipStdPar(
   cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false),
   cl::Hidden);
 
+static cl::opt<bool>
+    EnableAMDGPUAttributor("amdgpu-attributor-enable",
+                           cl::desc("Enable AMDGPUAttributorPass"),
+                           cl::init(true), cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -764,6 +769,8 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
         // module is partitioned for codegen.
         if (EnableLowerModuleLDS)
           PM.addPass(AMDGPULowerModuleLDSPass(*this));
+        if (EnableAMDGPUAttributor && Level != OptimizationLevel::O0)
+          PM.addPass(AMDGPUAttributorPass(*this));
       });
 
   PB.registerRegClassFilterParsingCallback(
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 98f9b3bc6aadad3..54dc1972d27619b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -46,7 +46,10 @@ struct GCNRegPressure {
 
   void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); }
 
+  /// \returns the SGPR32 pressure
   unsigned getSGPRNum() const { return Value[SGPR32]; }
+  /// \returns the aggregated ArchVGPR32, AccVGPR32 pressure dependent upon \p
+  /// UnifiedVGPRFile
   unsigned getVGPRNum(bool UnifiedVGPRFile) const {
     if (UnifiedVGPRFile) {
       return Value[AGPR32] ? alignTo(Value[VGPR32], 4) + Value[AGPR32]
@@ -54,6 +57,9 @@ struct GCNRegPressure {
     }
     return std::max(Value[VGPR32], Value[AGPR32]);
   }
+  /// \returns the ArchVGPR32 pressure
+  unsigned getArchVGPRNum() const { return Value[VGPR32]; }
+  /// \returns the AccVGPR32 pressure
   unsigned getAGPRNum() const { return Value[AGPR32]; }
 
   unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE],
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 155747551471e38..5d38cafd73dd95a 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -1048,7 +1048,7 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
               MachineFunctionProperties::Property::NoVRegs))
         continue;
 
-      if (ST->useRealTrue16Insts() && AMDGPU::isTrue16Inst(MI.getOpcode()) &&
+      if (ST->hasTrue16BitInsts() && AMDGPU::isTrue16Inst(MI.getOpcode()) &&
           !shouldShrinkTrue16(MI))
         continue;
 
diff --git a/llvm/lib/Target/Mips/Mips64InstrInfo.td b/llvm/lib/Target/Mips/Mips64InstrInfo.td
index cb9a0d44a7a72b9..f19eaf7a67f73c9 100644
--- a/llvm/lib/Target/Mips/Mips64InstrInfo.td
+++ b/llvm/lib/Target/Mips/Mips64InstrInfo.td
@@ -843,7 +843,7 @@ def : MipsPat<(i64 (anyext GPR32:$src)),
       ISA_MIPS3, GPR_64;
 def : MipsPat<(i64 (zext GPR32:$src)), (DSRL (DSLL64_32 GPR32:$src), 32)>,
       ISA_MIPS3, GPR_64;
-def : MipsPat<(i64 (sext GPR32:$src)), (SLL64_32 GPR32:$src)>, ISA_MIPS3,
+def : MipsPat<(i64 (sext_like GPR32:$src)), (SLL64_32 GPR32:$src)>, ISA_MIPS3,
       GPR_64;
 
 let AdditionalPredicates = [NotInMicroMips] in {
@@ -911,40 +911,40 @@ def : MipsPat<(atomic_store_64 GPR64:$v, addr:$a), (SD GPR64:$v, addr:$a)>,
 
 // Patterns used for matching away redundant sign extensions.
 // MIPS32 arithmetic instructions sign extend their result implicitly.
-def : MipsPat<(i64 (sext (i32 (add GPR32:$src, immSExt16:$imm16)))),
+def : MipsPat<(i64 (sext_like (i32 (add GPR32:$src, immSExt16:$imm16)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (ADDiu GPR32:$src, immSExt16:$imm16), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (add GPR32:$src, GPR32:$src2)))),
+def : MipsPat<(i64 (sext_like (i32 (add GPR32:$src, GPR32:$src2)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (ADDu GPR32:$src, GPR32:$src2), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (sub GPR32:$src, GPR32:$src2)))),
+def : MipsPat<(i64 (sext_like (i32 (sub GPR32:$src, GPR32:$src2)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (SUBu GPR32:$src, GPR32:$src2), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (mul GPR32:$src, GPR32:$src2)))),
+def : MipsPat<(i64 (sext_like (i32 (mul GPR32:$src, GPR32:$src2)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (MUL GPR32:$src, GPR32:$src2), sub_32)>, ISA_MIPS32_NOT_32R6_64R6;
-def : MipsPat<(i64 (sext (i32 (MipsMFHI ACC64:$src)))),
+def : MipsPat<(i64 (sext_like (i32 (MipsMFHI ACC64:$src)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (PseudoMFHI ACC64:$src), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (MipsMFLO ACC64:$src)))),
+def : MipsPat<(i64 (sext_like (i32 (MipsMFLO ACC64:$src)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (PseudoMFLO ACC64:$src), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (shl GPR32:$src, immZExt5:$imm5)))),
+def : MipsPat<(i64 (sext_like (i32 (shl GPR32:$src, immZExt5:$imm5)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (SLL GPR32:$src, immZExt5:$imm5), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (shl GPR32:$src, GPR32:$src2)))),
+def : MipsPat<(i64 (sext_like (i32 (shl GPR32:$src, GPR32:$src2)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (SLLV GPR32:$src, GPR32:$src2), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (srl GPR32:$src, immZExt5:$imm5)))),
+def : MipsPat<(i64 (sext_like (i32 (srl GPR32:$src, immZExt5:$imm5)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (SRL GPR32:$src, immZExt5:$imm5), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (srl GPR32:$src, GPR32:$src2)))),
+def : MipsPat<(i64 (sext_like (i32 (srl GPR32:$src, GPR32:$src2)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (SRLV GPR32:$src, GPR32:$src2), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (sra GPR32:$src, immZExt5:$imm5)))),
+def : MipsPat<(i64 (sext_like (i32 (sra GPR32:$src, immZExt5:$imm5)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (SRA GPR32:$src, immZExt5:$imm5), sub_32)>;
-def : MipsPat<(i64 (sext (i32 (sra GPR32:$src, GPR32:$src2)))),
+def : MipsPat<(i64 (sext_like (i32 (sra GPR32:$src, GPR32:$src2)))),
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
               (SRAV GPR32:$src, GPR32:$src2), sub_32)>;
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 059cfff1f7e6929..43a3fbf4d1306a9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -5920,6 +5920,8 @@ static SDValue PerformSETCCCombine(SDNode *N,
 static SDValue PerformEXTRACTCombine(SDNode *N,
                                      TargetLowering::DAGCombinerInfo &DCI) {
   SDValue Vector = N->getOperand(0);
+  if (Vector->getOpcode() == ISD::FREEZE)
+    Vector = Vector->getOperand(0);
   SDLoc DL(N);
   EVT VectorVT = Vector.getValueType();
   if (Vector->getOpcode() == ISD::LOAD && VectorVT.isSimple() &&
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index f1231b2b9baba28..8ff9f5a5a991e08 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -105,6 +105,11 @@ using namespace llvm;
 
 #define DEBUG_TYPE "ppc-lowering"
 
+static cl::opt<bool> DisableP10StoreForward(
+    "disable-p10-store-forward",
+    cl::desc("disable P10 store forward-friendly conversion"), cl::Hidden,
+    cl::init(false));
+
 static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc",
 cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden);
 
@@ -985,6 +990,14 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
 
     setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
     setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom);
+    // LE is P8+/64-bit so direct moves are supported and these operations
+    // are legal. The custom transformation requires 64-bit since we need a
+    // pair of stores that will cover a 128-bit load for P10.
+    if (!DisableP10StoreForward && isPPC64 && !Subtarget.isLittleEndian()) {
+      setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Custom);
+      setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
+      setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
+    }
 
     setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom);
     setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom);
@@ -11483,9 +11496,33 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT);
 
+  SDValue Val = Op.getOperand(0);
+  EVT ValVT = Val.getValueType();
+  // P10 hardware store forwarding requires that a single store contains all
+  // the data for the load. P10 is able to merge a pair of adjacent stores. Try
+  // to avoid load hit store on P10 when running binaries compiled for older
+  // processors by generating two mergeable scalar stores to forward with the
+  // vector load.
+  if (!DisableP10StoreForward && Subtarget.isPPC64() &&
+      !Subtarget.isLittleEndian() && ValVT.isInteger() &&
+      ValVT.getSizeInBits() <= 64) {
+    Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, Val);
+    EVT ShiftAmountTy = getShiftAmountTy(MVT::i64, DAG.getDataLayout());
+    SDValue ShiftBy = DAG.getConstant(
+        64 - Op.getValueType().getScalarSizeInBits(), dl, ShiftAmountTy);
+    Val = DAG.getNode(ISD::SHL, dl, MVT::i64, Val, ShiftBy);
+    SDValue Plus8 =
+        DAG.getNode(ISD::ADD, dl, PtrVT, FIdx, DAG.getConstant(8, dl, PtrVT));
+    SDValue Store2 =
+        DAG.getStore(DAG.getEntryNode(), dl, Val, Plus8, MachinePointerInfo());
+    SDValue Store = DAG.getStore(Store2, dl, Val, FIdx, MachinePointerInfo());
+    return DAG.getLoad(Op.getValueType(), dl, Store, FIdx,
+                       MachinePointerInfo());
+  }
+
   // Store the input value into Value#0 of the stack slot.
-  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx,
-                               MachinePointerInfo());
+  SDValue Store =
+      DAG.getStore(DAG.getEntryNode(), dl, Val, FIdx, MachinePointerInfo());
   // Load it out.
   return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo());
 }
diff --git a/llvm/lib/Target/RISCV/RISCVGISel.td b/llvm/lib/Target/RISCV/RISCVGISel.td
index 5f16ffb0a0248e8..84c00d31800ceef 100644
--- a/llvm/lib/Target/RISCV/RISCVGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVGISel.td
@@ -50,6 +50,12 @@ def GIAddrRegImm :
   GIComplexOperandMatcher<s32, "selectAddrRegImm">,
   GIComplexPatternEquiv<AddrRegImm>;
 
+// Convert from i32 immediate to i64 target immediate to make SelectionDAG type
+// checking happy so we can use ADDIW which expects an XLen immediate.
+def as_i64imm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i64);
+}]>;
+
 def gi_as_i64imm : GICustomOperandRenderer<"renderImm">,
   GISDNodeXFormEquiv<as_i64imm>;
 
@@ -157,3 +163,305 @@ let Predicates = [IsRV64] in {
 def : LdPat<load, LD, PtrVT>;
 def : StPat<store, SD, GPR, PtrVT>;
 }
+
+//===----------------------------------------------------------------------===//
+// RV64 i32 patterns not used by SelectionDAG
+//===----------------------------------------------------------------------===//
+
+def simm12i32 : ImmLeaf<i32, [{return isInt<12>(Imm);}]>;
+
+def zext_is_sext : PatFrag<(ops node:$src), (zext node:$src), [{
+  KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0), 0);
+  return Known.isNonNegative();
+}]>;
+
+let Predicates = [IsRV64] in {
+def : LdPat<sextloadi8, LB, i32>;
+def : LdPat<extloadi8, LBU, i32>; // Prefer unsigned due to no c.lb in Zcb.
+def : LdPat<sextloadi16, LH, i32>;
+def : LdPat<extloadi16, LH, i32>;
+def : LdPat<zextloadi8, LBU, i32>;
+def : LdPat<zextloadi16, LHU, i32>;
+
+def : StPat<truncstorei8, SB, GPR, i32>;
+def : StPat<truncstorei16, SH, GPR, i32>;
+
+def : Pat<(anyext GPR:$src), (COPY GPR:$src)>;
+def : Pat<(sext GPR:$src), (ADDIW GPR:$src, 0)>;
+def : Pat<(trunc GPR:$src), (COPY GPR:$src)>;
+
+def : PatGprGpr<add, ADDW, i32, i32>;
+def : PatGprGpr<sub, SUBW, i32, i32>;
+def : PatGprGpr<and, AND, i32, i32>;
+def : PatGprGpr<or, OR, i32, i32>;
+def : PatGprGpr<xor, XOR, i32, i32>;
+def : PatGprGpr<shiftopw<shl>, SLLW, i32, i64>;
+def : PatGprGpr<shiftopw<srl>, SRLW, i32, i64>;
+def : PatGprGpr<shiftopw<sra>, SRAW, i32, i64>;
+
+def : Pat<(i32 (add GPR:$rs1, simm12i32:$imm)),
+          (ADDIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (and GPR:$rs1, simm12i32:$imm)),
+          (ANDI GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (or GPR:$rs1, simm12i32:$imm)),
+          (ORI GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (xor GPR:$rs1, simm12i32:$imm)),
+          (XORI GPR:$rs1, (i64 (as_i64imm $imm)))>;
+
+def : PatGprImm<shl, SLLIW, uimm5, i32>;
+def : PatGprImm<srl, SRLIW, uimm5, i32>;
+def : PatGprImm<sra, SRAIW, uimm5, i32>;
+
+def : Pat<(i32 (and GPR:$rs, TrailingOnesMask:$mask)),
+          (SRLI (i32 (SLLI $rs, (i64 (XLenSubTrailingOnes $mask)))),
+                (i64 (XLenSubTrailingOnes $mask)))>;
+
+// Use sext if the sign bit of the input is 0.
+def : Pat<(zext_is_sext GPR:$src), (ADDIW GPR:$src, 0)>;
+}
+
+let Predicates = [IsRV64, NotHasStdExtZba] in {
+def : Pat<(zext GPR:$src), (SRLI (i64 (SLLI GPR:$src, 32)), 32)>;
+
+// If we're shifting a 32-bit zero extended value left by 0-31 bits, use 2
+// shifts instead of 3. This can occur when unsigned is used to index an array.
+def : Pat<(shl (zext GPR:$rs), uimm5:$shamt),
+          (SRLI (i64 (SLLI GPR:$rs, 32)), (ImmSubFrom32 uimm5:$shamt))>;
+}
+
+//===----------------------------------------------------------------------===//
+// M RV64 i32 legalization patterns.
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtZmmul, IsRV64] in {
+def : PatGprGpr<mul, MULW, i32, i32>;
+}
+
+let Predicates = [HasStdExtM, IsRV64] in {
+def : PatGprGpr<sdiv, DIVW, i32, i32>;
+def : PatGprGpr<udiv, DIVUW, i32, i32>;
+def : PatGprGpr<srem, REMW, i32, i32>;
+def : PatGprGpr<urem, REMUW, i32, i32>;
+}
+
+//===----------------------------------------------------------------------===//
+// Atomic RV64 i32 patterns not used by SelectionDAG
+//===----------------------------------------------------------------------===//
+
+class PatGprGprA<SDPatternOperator OpNode, RVInst Inst, ValueType vt>
+    : Pat<(vt (OpNode (XLenVT GPR:$rs1), (vt GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2)>;
+
+multiclass AMOPat2<string AtomicOp, string BaseInst, ValueType vt = XLenVT,
+                   list<Predicate> ExtraPreds = []> {
+let Predicates = !listconcat([HasStdExtA, NotHasStdExtZtso], ExtraPreds) in {
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_monotonic"),
+                   !cast<RVInst>(BaseInst), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acquire"),
+                   !cast<RVInst>(BaseInst#"_AQ"), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_release"),
+                   !cast<RVInst>(BaseInst#"_RL"), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acq_rel"),
+                   !cast<RVInst>(BaseInst#"_AQ_RL"), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_seq_cst"),
+                   !cast<RVInst>(BaseInst#"_AQ_RL"), vt>;
+}
+let Predicates = !listconcat([HasStdExtA, HasStdExtZtso], ExtraPreds) in {
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_monotonic"),
+                   !cast<RVInst>(BaseInst), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acquire"),
+                   !cast<RVInst>(BaseInst), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_release"),
+                   !cast<RVInst>(BaseInst), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acq_rel"),
+                   !cast<RVInst>(BaseInst), vt>;
+  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_seq_cst"),
+                   !cast<RVInst>(BaseInst), vt>;
+}
+}
+
+defm : AMOPat2<"atomic_swap_i32", "AMOSWAP_W", i32>;
+defm : AMOPat2<"atomic_load_add_i32", "AMOADD_W", i32>;
+defm : AMOPat2<"atomic_load_and_i32", "AMOAND_W", i32>;
+defm : AMOPat2<"atomic_load_or_i32", "AMOOR_W", i32>;
+defm : AMOPat2<"atomic_load_xor_i32", "AMOXOR_W", i32>;
+defm : AMOPat2<"atomic_load_max_i32", "AMOMAX_W", i32>;
+defm : AMOPat2<"atomic_load_min_i32", "AMOMIN_W", i32>;
+defm : AMOPat2<"atomic_load_umax_i32", "AMOMAXU_W", i32>;
+defm : AMOPat2<"atomic_load_umin_i32", "AMOMINU_W", i32>;
+
+let Predicates = [HasStdExtA, IsRV64] in
+defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32, i32>;
+
+let Predicates = [HasAtomicLdSt] in {
+  def : LdPat<atomic_load_8,  LB, i32>;
+  def : LdPat<atomic_load_16, LH, i32>;
+  def : LdPat<atomic_load_32, LW, i32>;
+
+  def : StPat<atomic_store_8,  SB, GPR, i32>;
+  def : StPat<atomic_store_16, SH, GPR, i32>;
+  def : StPat<atomic_store_32, SW, GPR, i32>;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Zb* RV64 i32 patterns not used by SelectionDAG.
+//===----------------------------------------------------------------------===//
+
+def zexti16i32 : ComplexPattern<i32, 1, "selectZExtBits<16>">;
+def zexti8i32 : ComplexPattern<i32, 1, "selectZExtBits<8>">;
+
+def BCLRMaski32 : ImmLeaf<i32, [{
+  return !isInt<12>(Imm) && isPowerOf2_32(~Imm);
+}]>;
+def SingleBitSetMaski32 : ImmLeaf<i32, [{
+  return !isInt<12>(Imm) && isPowerOf2_32(Imm);
+}]>;
+
+let Predicates = [HasStdExtZbb, IsRV64] in {
+def : PatGpr<ctlz, CLZW, i32>;
+def : PatGpr<cttz, CTZW, i32>;
+def : PatGpr<ctpop, CPOPW, i32>;
+
+def : Pat<(i32 (sext_inreg GPR:$rs1, i8)), (SEXT_B GPR:$rs1)>;
+def : Pat<(i32 (sext_inreg GPR:$rs1, i16)), (SEXT_H GPR:$rs1)>;
+
+def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (ZEXT_H_RV64 GPR:$rs)>;
+} // Predicates = [HasStdExtZbb, IsRV64]
+
+let Predicates = [HasStdExtZbkb, NoStdExtZbb, IsRV64] in {
+def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (PACKW GPR:$rs, (XLenVT X0))>;
+}
+
+let Predicates = [HasStdExtZbbOrZbkb, IsRV64] in {
+def : Pat<(i32 (and GPR:$rs1, (not GPR:$rs2))), (ANDN GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (or  GPR:$rs1, (not GPR:$rs2))), (ORN  GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (xor GPR:$rs1, (not GPR:$rs2))), (XNOR GPR:$rs1, GPR:$rs2)>;
+
+def : PatGprGpr<shiftopw<rotl>, ROLW, i32, i64>;
+def : PatGprGpr<shiftopw<rotr>, RORW, i32, i64>;
+def : PatGprImm<rotr, RORIW, uimm5, i32>;
+
+def : Pat<(i32 (rotl GPR:$rs1, uimm5:$rs2)),
+          (RORIW GPR:$rs1, (ImmSubFrom32 uimm5:$rs2))>;
+} // Predicates = [HasStdExtZbbOrZbkb, IsRV64]
+
+let Predicates = [HasStdExtZbkb, IsRV64] in {
+def : Pat<(or (and (shl GPR:$rs2, (i64 8)), 0xFFFF),
+              (zexti8i32 (i32 GPR:$rs1))),
+          (PACKH GPR:$rs1, GPR:$rs2)>;
+def : Pat<(or (shl (zexti8i32 (i32 GPR:$rs2)), (i64 8)),
+              (zexti8i32 (i32 GPR:$rs1))),
+          (PACKH GPR:$rs1, GPR:$rs2)>;
+def : Pat<(and (anyext (or (shl GPR:$rs2, (XLenVT 8)),
+                           (zexti8i32 (i32 GPR:$rs1)))), 0xFFFF),
+          (PACKH GPR:$rs1, GPR:$rs2)>;
+
+def : Pat<(i32 (or (shl GPR:$rs2, (i64 16)), (zexti16i32 (i32 GPR:$rs1)))),
+          (PACKW GPR:$rs1, GPR:$rs2)>;
+} // Predicates = [HasStdExtZbkb, IsRV64]
+
+let Predicates = [HasStdExtZba, IsRV64] in {
+def : Pat<(shl (i64 (zext i32:$rs1)), uimm5:$shamt),
+          (SLLI_UW GPR:$rs1, uimm5:$shamt)>;
+
+def : Pat<(i64 (add_like_non_imm12 (zext GPR:$rs1), GPR:$rs2)),
+          (ADD_UW GPR:$rs1, GPR:$rs2)>;
+def : Pat<(zext GPR:$src), (ADD_UW GPR:$src, (XLenVT X0))>;
+
+foreach i = {1,2,3} in {
+  defvar shxadd = !cast<Instruction>("SH"#i#"ADD");
+  def : Pat<(i32 (add_like_non_imm12 (shl GPR:$rs1, (i64 i)), GPR:$rs2)),
+            (shxadd GPR:$rs1, GPR:$rs2)>;
+  def : Pat<(i32 (riscv_shl_add GPR:$rs1, (i32 i), GPR:$rs2)),
+            (shxadd GPR:$rs1, GPR:$rs2)>;
+}
+}
+
+let Predicates = [HasStdExtZbs, IsRV64] in {
+def : Pat<(i32 (and (not (shiftop<shl> 1, (i64 GPR:$rs2))), GPR:$rs1)),
+          (BCLR GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (and (rotl -2, (i64 GPR:$rs2)), GPR:$rs1)),
+          (BCLR GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (or (shiftop<shl> 1, (i64 GPR:$rs2)), GPR:$rs1)),
+          (BSET GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (xor (shiftop<shl> 1, (i64 GPR:$rs2)), GPR:$rs1)),
+          (BINV GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (and (shiftop<srl> GPR:$rs1, (i64 GPR:$rs2)), 1)),
+          (BEXT GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i64 (and (anyext (i32 (shiftop<srl> GPR:$rs1, (i64 GPR:$rs2)))), 1)),
+          (BEXT GPR:$rs1, GPR:$rs2)>;
+
+def : Pat<(i32 (shiftop<shl> 1, (i64 GPR:$rs2))),
+          (BSET (XLenVT X0), GPR:$rs2)>;
+def : Pat<(i32 (not (shiftop<shl> -1, (i64 GPR:$rs2)))),
+          (ADDI (i32 (BSET (XLenVT X0), GPR:$rs2)), -1)>;
+
+def : Pat<(i32 (and (srl GPR:$rs1, uimm5:$shamt), (i32 1))),
+          (BEXTI GPR:$rs1, uimm5:$shamt)>;
+
+def : Pat<(i32 (and GPR:$rs1, BCLRMaski32:$mask)),
+          (BCLRI GPR:$rs1, (i64 (BCLRXForm $mask)))>;
+def : Pat<(i32 (or GPR:$rs1, SingleBitSetMaski32:$mask)),
+          (BSETI GPR:$rs1, (i64 (SingleBitSetMaskToIndex $mask)))>;
+def : Pat<(i32 (xor GPR:$rs1, SingleBitSetMaski32:$mask)),
+          (BINVI GPR:$rs1, (i64 (SingleBitSetMaskToIndex $mask)))>;
+} // Predicates = [HasStdExtZbs, IsRV64]
+
+//===----------------------------------------------------------------------===//
+// XTHead RV64 i32 patterns not used by SelectionDAG.
+//===----------------------------------------------------------------------===//
+
+def sexti16i32 : ComplexPattern<i32, 1, "selectSExtBits<16>">;
+
+let Predicates = [HasVendorXTHeadMemIdx, IsRV64] in {
+defm : StoreUpdatePat<post_truncsti8, TH_SBIA, i32>;
+defm : StoreUpdatePat<pre_truncsti8, TH_SBIB, i32>;
+defm : StoreUpdatePat<post_truncsti16, TH_SHIA, i32>;
+defm : StoreUpdatePat<pre_truncsti16, TH_SHIB, i32>;
+
+defm : StIdxPat<truncstorei8, TH_SRB, GPR, i32>;
+defm : StIdxPat<truncstorei16, TH_SRH, GPR, i32>;
+
+defm : StZextIdxPat<truncstorei8, TH_SURB, GPR, i32>;
+defm : StZextIdxPat<truncstorei16, TH_SURH, GPR, i32>;
+defm : StZextIdxPat<store, TH_SURW, GPR, i32>;
+}
+
+let Predicates = [HasVendorXTHeadCondMov, IsRV64] in {
+def : Pat<(select (XLenVT GPR:$cond), (i32 GPR:$a), (i32 GPR:$b)),
+          (TH_MVEQZ GPR:$a, GPR:$b, GPR:$cond)>;
+def : Pat<(select (XLenVT GPR:$cond), (i32 GPR:$a), (i32 0)),
+          (TH_MVEQZ GPR:$a, (XLenVT X0), GPR:$cond)>;
+def : Pat<(select (XLenVT GPR:$cond), (i32 0), (i32 GPR:$b)),
+          (TH_MVNEZ GPR:$b, (XLenVT X0), GPR:$cond)>;
+
+def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 GPR:$b)),
+          (TH_MVNEZ GPR:$a, GPR:$b, GPR:$cond)>;
+def : Pat<(select (riscv_setne (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 GPR:$b)),
+          (TH_MVEQZ GPR:$a, GPR:$b, GPR:$cond)>;
+def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 0)),
+          (TH_MVNEZ GPR:$a, (XLenVT X0), GPR:$cond)>;
+def : Pat<(select (riscv_setne (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 0)),
+          (TH_MVEQZ GPR:$a, (XLenVT X0), GPR:$cond)>;
+def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 0), (i32 GPR:$b)),
+          (TH_MVEQZ GPR:$b, (XLenVT X0), GPR:$cond)>;
+def : Pat<(select (riscv_setne (XLenVT GPR:$cond)),  (i32 0), (i32 GPR:$b)),
+          (TH_MVNEZ GPR:$b, (XLenVT X0), GPR:$cond)>;
+} // Predicates = [HasVendorXTHeadCondMov]
+
+let Predicates = [HasVendorXTHeadMac, IsRV64] in {
+// mulaw, mulsw are available only in RV64.
+def : Pat<(i32 (add GPR:$rd, (mul GPR:$rs1, GPR:$rs2))),
+          (TH_MULAW GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (sub GPR:$rd, (mul GPR:$rs1, GPR:$rs2))),
+          (TH_MULSW GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+// mulah, mulsh produce a sign-extended result.
+def : Pat<(i32 (add GPR:$rd,
+                    (mul (sexti16i32 (i32 GPR:$rs1)),
+                         (sexti16i32 (i32 GPR:$rs2))))),
+          (TH_MULAH GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+def : Pat<(i32 (sub GPR:$rd,
+                    (mul (sexti16i32 (i32 GPR:$rs1)),
+                         (sexti16i32 (i32 GPR:$rs2))))),
+          (TH_MULSH GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 872c288f5ad8cda..3beaa5198259d0e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -268,11 +268,11 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::LOAD, MVT::i32, Custom);
     setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
                        MVT::i32, Custom);
-    setOperationAction({ISD::UADDO, ISD::USUBO, ISD::UADDSAT}, MVT::i32,
-                       Custom);
+    setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Custom);
     if (!Subtarget.hasStdExtZbb())
-      setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, MVT::i32,
-                         Custom);
+      setOperationAction(
+          {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT}, MVT::i32,
+          Custom);
     setOperationAction(ISD::SADDO, MVT::i32, Custom);
   }
   if (!Subtarget.hasStdExtZmmul()) {
@@ -12173,20 +12173,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::UADDSAT:
   case ISD::USUBSAT: {
     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
-           "Unexpected custom legalisation");
-    if (Subtarget.hasStdExtZbb()) {
-      // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
-      // sign extend allows overflow of the lower 32 bits to be detected on
-      // the promoted size.
-      SDValue LHS =
-          DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
-      SDValue RHS =
-          DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
-      SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
-      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
-      return;
-    }
-
+           !Subtarget.hasStdExtZbb() && "Unexpected custom legalisation");
     // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
     // promotion for UADDO/USUBO.
     Results.push_back(expandAddSubSat(N, DAG));
@@ -21365,8 +21352,9 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
   return true;
 }
 
-bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *DI,
-                                                           LoadInst *LI) const {
+bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
+    IntrinsicInst *DI, LoadInst *LI,
+    SmallVectorImpl<Instruction *> &DeadInsts) const {
   assert(LI->isSimple());
   IRBuilder<> Builder(LI);
 
@@ -21415,8 +21403,9 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *DI,
   return true;
 }
 
-bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(IntrinsicInst *II,
-                                                          StoreInst *SI) const {
+bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
+    IntrinsicInst *II, StoreInst *SI,
+    SmallVectorImpl<Instruction *> &DeadInsts) const {
   assert(SI->isSimple());
   IRBuilder<> Builder(SI);
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index d1d0760d8ffd1ce..2298998b47357de 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -879,11 +879,13 @@ class RISCVTargetLowering : public TargetLowering {
   bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
                              unsigned Factor) const override;
 
-  bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II,
-                                        LoadInst *LI) const override;
+  bool lowerDeinterleaveIntrinsicToLoad(
+      IntrinsicInst *II, LoadInst *LI,
+      SmallVectorImpl<Instruction *> &DeadInsts) const override;
 
-  bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II,
-                                       StoreInst *SI) const override;
+  bool lowerInterleaveIntrinsicToStore(
+      IntrinsicInst *II, StoreInst *SI,
+      SmallVectorImpl<Instruction *> &DeadInsts) const override;
 
   bool supportKCFIBundles() const override { return true; }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index cebe95ca1650697..2dd7240ef941a73 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1206,16 +1206,13 @@ def assertsexti32 : PatFrag<(ops node:$src), (assertsext node:$src), [{
   return cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32);
 }]>;
 def sexti16 : ComplexPattern<XLenVT, 1, "selectSExtBits<16>">;
-def sexti16i32 : ComplexPattern<i32, 1, "selectSExtBits<16>">;
 def sexti32 : ComplexPattern<i64, 1, "selectSExtBits<32>">;
 def assertzexti32 : PatFrag<(ops node:$src), (assertzext node:$src), [{
   return cast<VTSDNode>(N->getOperand(1))->getVT().bitsLE(MVT::i32);
 }]>;
 def zexti32 : ComplexPattern<i64, 1, "selectZExtBits<32>">;
 def zexti16 : ComplexPattern<XLenVT, 1, "selectZExtBits<16>">;
-def zexti16i32 : ComplexPattern<i32, 1, "selectZExtBits<16>">;
 def zexti8 : ComplexPattern<XLenVT, 1, "selectZExtBits<8>">;
-def zexti8i32 : ComplexPattern<i32, 1, "selectZExtBits<8>">;
 
 def ext : PatFrags<(ops node:$A), [(sext node:$A), (zext node:$A)]>;
 
@@ -1961,77 +1958,6 @@ def : Pat<(binop_allwusers<add> GPR:$rs1, (AddiPair:$rs2)),
                  (AddiPairImmSmall AddiPair:$rs2))>;
 }
 
-//===----------------------------------------------------------------------===//
-// Experimental RV64 i32 legalization patterns.
-//===----------------------------------------------------------------------===//
-
-def simm12i32 : ImmLeaf<i32, [{return isInt<12>(Imm);}]>;
-
-// Convert from i32 immediate to i64 target immediate to make SelectionDAG type
-// checking happy so we can use ADDIW which expects an XLen immediate.
-def as_i64imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i64);
-}]>;
-
-def zext_is_sext : PatFrag<(ops node:$src), (zext node:$src), [{
-  KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0), 0);
-  return Known.isNonNegative();
-}]>;
-
-let Predicates = [IsRV64] in {
-def : LdPat<sextloadi8, LB, i32>;
-def : LdPat<extloadi8, LBU, i32>; // Prefer unsigned due to no c.lb in Zcb.
-def : LdPat<sextloadi16, LH, i32>;
-def : LdPat<extloadi16, LH, i32>;
-def : LdPat<zextloadi8, LBU, i32>;
-def : LdPat<zextloadi16, LHU, i32>;
-
-def : StPat<truncstorei8, SB, GPR, i32>;
-def : StPat<truncstorei16, SH, GPR, i32>;
-
-def : Pat<(anyext GPR:$src), (COPY GPR:$src)>;
-def : Pat<(sext GPR:$src), (ADDIW GPR:$src, 0)>;
-def : Pat<(trunc GPR:$src), (COPY GPR:$src)>;
-
-def : PatGprGpr<add, ADDW, i32, i32>;
-def : PatGprGpr<sub, SUBW, i32, i32>;
-def : PatGprGpr<and, AND, i32, i32>;
-def : PatGprGpr<or, OR, i32, i32>;
-def : PatGprGpr<xor, XOR, i32, i32>;
-def : PatGprGpr<shiftopw<shl>, SLLW, i32, i64>;
-def : PatGprGpr<shiftopw<srl>, SRLW, i32, i64>;
-def : PatGprGpr<shiftopw<sra>, SRAW, i32, i64>;
-
-def : Pat<(i32 (add GPR:$rs1, simm12i32:$imm)),
-          (ADDIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
-def : Pat<(i32 (and GPR:$rs1, simm12i32:$imm)),
-          (ANDI GPR:$rs1, (i64 (as_i64imm $imm)))>;
-def : Pat<(i32 (or GPR:$rs1, simm12i32:$imm)),
-          (ORI GPR:$rs1, (i64 (as_i64imm $imm)))>;
-def : Pat<(i32 (xor GPR:$rs1, simm12i32:$imm)),
-          (XORI GPR:$rs1, (i64 (as_i64imm $imm)))>;
-
-def : PatGprImm<shl, SLLIW, uimm5, i32>;
-def : PatGprImm<srl, SRLIW, uimm5, i32>;
-def : PatGprImm<sra, SRAIW, uimm5, i32>;
-
-def : Pat<(i32 (and GPR:$rs, TrailingOnesMask:$mask)),
-          (SRLI (i32 (SLLI $rs, (i64 (XLenSubTrailingOnes $mask)))),
-                (i64 (XLenSubTrailingOnes $mask)))>;
-
-// Use sext if the sign bit of the input is 0.
-def : Pat<(zext_is_sext GPR:$src), (ADDIW GPR:$src, 0)>;
-}
-
-let Predicates = [IsRV64, NotHasStdExtZba] in {
-def : Pat<(zext GPR:$src), (SRLI (i64 (SLLI GPR:$src, 32)), 32)>;
-
-// If we're shifting a 32-bit zero extended value left by 0-31 bits, use 2
-// shifts instead of 3. This can occur when unsigned is used to index an array.
-def : Pat<(shl (zext GPR:$rs), uimm5:$shamt),
-          (SRLI (i64 (SLLI GPR:$rs, 32)), (ImmSubFrom32 uimm5:$shamt))>;
-}
-
 //===----------------------------------------------------------------------===//
 // Standard extensions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
index 493e1a5fdc74a20..10e919c902d518d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
@@ -386,62 +386,3 @@ def : Pat<(int_riscv_masked_cmpxchg_i64
           (PseudoMaskedCmpXchg32
             GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering)>;
 } // Predicates = [HasStdExtA, IsRV64]
-
-//===----------------------------------------------------------------------===//
-// Experimental RV64 i32 legalization patterns.
-//===----------------------------------------------------------------------===//
-
-class PatGprGprA<SDPatternOperator OpNode, RVInst Inst, ValueType vt>
-    : Pat<(vt (OpNode (XLenVT GPR:$rs1), (vt GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2)>;
-
-multiclass AMOPat2<string AtomicOp, string BaseInst, ValueType vt = XLenVT,
-                   list<Predicate> ExtraPreds = []> {
-let Predicates = !listconcat([HasStdExtA, NotHasStdExtZtso], ExtraPreds) in {
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_monotonic"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acquire"),
-                   !cast<RVInst>(BaseInst#"_AQ"), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_release"),
-                   !cast<RVInst>(BaseInst#"_RL"), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acq_rel"),
-                   !cast<RVInst>(BaseInst#"_AQ_RL"), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_seq_cst"),
-                   !cast<RVInst>(BaseInst#"_AQ_RL"), vt>;
-}
-let Predicates = !listconcat([HasStdExtA, HasStdExtZtso], ExtraPreds) in {
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_monotonic"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acquire"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_release"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_acq_rel"),
-                   !cast<RVInst>(BaseInst), vt>;
-  def : PatGprGprA<!cast<PatFrag>(AtomicOp#"_seq_cst"),
-                   !cast<RVInst>(BaseInst), vt>;
-}
-}
-
-defm : AMOPat2<"atomic_swap_i32", "AMOSWAP_W", i32>;
-defm : AMOPat2<"atomic_load_add_i32", "AMOADD_W", i32>;
-defm : AMOPat2<"atomic_load_and_i32", "AMOAND_W", i32>;
-defm : AMOPat2<"atomic_load_or_i32", "AMOOR_W", i32>;
-defm : AMOPat2<"atomic_load_xor_i32", "AMOXOR_W", i32>;
-defm : AMOPat2<"atomic_load_max_i32", "AMOMAX_W", i32>;
-defm : AMOPat2<"atomic_load_min_i32", "AMOMIN_W", i32>;
-defm : AMOPat2<"atomic_load_umax_i32", "AMOMAXU_W", i32>;
-defm : AMOPat2<"atomic_load_umin_i32", "AMOMINU_W", i32>;
-
-let Predicates = [HasStdExtA, IsRV64] in
-defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32, i32>;
-
-let Predicates = [HasAtomicLdSt] in {
-  def : LdPat<atomic_load_8,  LB, i32>;
-  def : LdPat<atomic_load_16, LH, i32>;
-  def : LdPat<atomic_load_32, LW, i32>;
-
-  def : StPat<atomic_store_8,  SB, GPR, i32>;
-  def : StPat<atomic_store_16, SH, GPR, i32>;
-  def : StPat<atomic_store_32, SW, GPR, i32>;
-}
-
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
index 8a2b32081dc5fa2..046e655c4a8c0a8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
@@ -114,18 +114,3 @@ let Predicates = [HasStdExtZmmul, IsRV64, NotHasStdExtZba] in {
 def : Pat<(i64 (mul (and GPR:$rs1, 0xffffffff), (and GPR:$rs2, 0xffffffff))),
           (MULHU (i64 (SLLI GPR:$rs1, 32)), (i64 (SLLI GPR:$rs2, 32)))>;
 } // Predicates = [HasStdExtZmmul, IsRV64, NotHasStdExtZba]
-
-//===----------------------------------------------------------------------===//
-// Experimental RV64 i32 legalization patterns.
-//===----------------------------------------------------------------------===//
-
-let Predicates = [HasStdExtZmmul, IsRV64] in {
-def : PatGprGpr<mul, MULW, i32, i32>;
-}
-
-let Predicates = [HasStdExtM, IsRV64] in {
-def : PatGprGpr<sdiv, DIVW, i32, i32>;
-def : PatGprGpr<udiv, DIVUW, i32, i32>;
-def : PatGprGpr<srem, REMW, i32, i32>;
-def : PatGprGpr<urem, REMUW, i32, i32>;
-}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index 9aca4aeae966b54..555bddc740678d1 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -845,60 +845,3 @@ defm : StoreUpdatePat<pre_truncsti32, TH_SWIB, i64>;
 defm : StoreUpdatePat<post_store, TH_SDIA, i64>;
 defm : StoreUpdatePat<pre_store, TH_SDIB, i64>;
 }
-
-//===----------------------------------------------------------------------===//
-// Experimental RV64 i32 legalization patterns.
-//===----------------------------------------------------------------------===//
-
-let Predicates = [HasVendorXTHeadMemIdx, IsRV64] in {
-defm : StoreUpdatePat<post_truncsti8, TH_SBIA, i32>;
-defm : StoreUpdatePat<pre_truncsti8, TH_SBIB, i32>;
-defm : StoreUpdatePat<post_truncsti16, TH_SHIA, i32>;
-defm : StoreUpdatePat<pre_truncsti16, TH_SHIB, i32>;
-
-defm : StIdxPat<truncstorei8, TH_SRB, GPR, i32>;
-defm : StIdxPat<truncstorei16, TH_SRH, GPR, i32>;
-
-defm : StZextIdxPat<truncstorei8, TH_SURB, GPR, i32>;
-defm : StZextIdxPat<truncstorei16, TH_SURH, GPR, i32>;
-defm : StZextIdxPat<store, TH_SURW, GPR, i32>;
-}
-
-let Predicates = [HasVendorXTHeadCondMov, IsRV64] in {
-def : Pat<(select (XLenVT GPR:$cond), (i32 GPR:$a), (i32 GPR:$b)),
-          (TH_MVEQZ GPR:$a, GPR:$b, GPR:$cond)>;
-def : Pat<(select (XLenVT GPR:$cond), (i32 GPR:$a), (i32 0)),
-          (TH_MVEQZ GPR:$a, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (XLenVT GPR:$cond), (i32 0), (i32 GPR:$b)),
-          (TH_MVNEZ GPR:$b, (XLenVT X0), GPR:$cond)>;
-
-def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 GPR:$b)),
-          (TH_MVNEZ GPR:$a, GPR:$b, GPR:$cond)>;
-def : Pat<(select (riscv_setne (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 GPR:$b)),
-          (TH_MVEQZ GPR:$a, GPR:$b, GPR:$cond)>;
-def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 0)),
-          (TH_MVNEZ GPR:$a, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (riscv_setne (XLenVT GPR:$cond)), (i32 GPR:$a), (i32 0)),
-          (TH_MVEQZ GPR:$a, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (riscv_seteq (XLenVT GPR:$cond)), (i32 0), (i32 GPR:$b)),
-          (TH_MVEQZ GPR:$b, (XLenVT X0), GPR:$cond)>;
-def : Pat<(select (riscv_setne (XLenVT GPR:$cond)),  (i32 0), (i32 GPR:$b)),
-          (TH_MVNEZ GPR:$b, (XLenVT X0), GPR:$cond)>;
-} // Predicates = [HasVendorXTHeadCondMov]
-
-let Predicates = [HasVendorXTHeadMac, IsRV64] in {
-// mulaw, mulsw are available only in RV64.
-def : Pat<(i32 (add GPR:$rd, (mul GPR:$rs1, GPR:$rs2))),
-          (TH_MULAW GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (sub GPR:$rd, (mul GPR:$rs1, GPR:$rs2))),
-          (TH_MULSW GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-// mulah, mulsh produce a sign-extended result.
-def : Pat<(i32 (add GPR:$rd,
-                    (mul (sexti16i32 (i32 GPR:$rs1)),
-                         (sexti16i32 (i32 GPR:$rs2))))),
-          (TH_MULAH GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (sub GPR:$rd,
-                    (mul (sexti16i32 (i32 GPR:$rs1)),
-                         (sexti16i32 (i32 GPR:$rs2))))),
-          (TH_MULSH GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index d5389deb4303ad2..8003bd98b8706a6 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -742,104 +742,3 @@ let Predicates = [HasStdExtZbkx] in {
 def : PatGprGpr<int_riscv_xperm4, XPERM4>;
 def : PatGprGpr<int_riscv_xperm8, XPERM8>;
 } // Predicates = [HasStdExtZbkx]
-
-//===----------------------------------------------------------------------===//
-// Experimental RV64 i32 legalization patterns.
-//===----------------------------------------------------------------------===//
-
-def BCLRMaski32 : ImmLeaf<i32, [{
-  return !isInt<12>(Imm) && isPowerOf2_32(~Imm);
-}]>;
-def SingleBitSetMaski32 : ImmLeaf<i32, [{
-  return !isInt<12>(Imm) && isPowerOf2_32(Imm);
-}]>;
-
-let Predicates = [HasStdExtZbb, IsRV64] in {
-def : PatGpr<ctlz, CLZW, i32>;
-def : PatGpr<cttz, CTZW, i32>;
-def : PatGpr<ctpop, CPOPW, i32>;
-
-def : Pat<(i32 (sext_inreg GPR:$rs1, i8)), (SEXT_B GPR:$rs1)>;
-def : Pat<(i32 (sext_inreg GPR:$rs1, i16)), (SEXT_H GPR:$rs1)>;
-
-def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (ZEXT_H_RV64 GPR:$rs)>;
-} // Predicates = [HasStdExtZbb, IsRV64]
-
-let Predicates = [HasStdExtZbkb, NoStdExtZbb, IsRV64] in {
-def : Pat<(i32 (and GPR:$rs, 0xFFFF)), (PACKW GPR:$rs, (XLenVT X0))>;
-}
-
-let Predicates = [HasStdExtZbbOrZbkb, IsRV64] in {
-def : Pat<(i32 (and GPR:$rs1, (not GPR:$rs2))), (ANDN GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (or  GPR:$rs1, (not GPR:$rs2))), (ORN  GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (xor GPR:$rs1, (not GPR:$rs2))), (XNOR GPR:$rs1, GPR:$rs2)>;
-
-def : PatGprGpr<shiftopw<rotl>, ROLW, i32, i64>;
-def : PatGprGpr<shiftopw<rotr>, RORW, i32, i64>;
-def : PatGprImm<rotr, RORIW, uimm5, i32>;
-
-def : Pat<(i32 (rotl GPR:$rs1, uimm5:$rs2)),
-          (RORIW GPR:$rs1, (ImmSubFrom32 uimm5:$rs2))>;
-} // Predicates = [HasStdExtZbbOrZbkb, IsRV64]
-
-let Predicates = [HasStdExtZbkb, IsRV64] in {
-def : Pat<(or (and (shl GPR:$rs2, (i64 8)), 0xFFFF),
-              (zexti8i32 (i32 GPR:$rs1))),
-          (PACKH GPR:$rs1, GPR:$rs2)>;
-def : Pat<(or (shl (zexti8i32 (i32 GPR:$rs2)), (i64 8)),
-              (zexti8i32 (i32 GPR:$rs1))),
-          (PACKH GPR:$rs1, GPR:$rs2)>;
-def : Pat<(and (anyext (or (shl GPR:$rs2, (XLenVT 8)),
-                           (zexti8i32 (i32 GPR:$rs1)))), 0xFFFF),
-          (PACKH GPR:$rs1, GPR:$rs2)>;
-
-def : Pat<(i32 (or (shl GPR:$rs2, (i64 16)), (zexti16i32 (i32 GPR:$rs1)))),
-          (PACKW GPR:$rs1, GPR:$rs2)>;
-} // Predicates = [HasStdExtZbkb, IsRV64]
-
-let Predicates = [HasStdExtZba, IsRV64] in {
-def : Pat<(shl (i64 (zext i32:$rs1)), uimm5:$shamt),
-          (SLLI_UW GPR:$rs1, uimm5:$shamt)>;
-
-def : Pat<(i64 (add_like_non_imm12 (zext GPR:$rs1), GPR:$rs2)),
-          (ADD_UW GPR:$rs1, GPR:$rs2)>;
-def : Pat<(zext GPR:$src), (ADD_UW GPR:$src, (XLenVT X0))>;
-
-foreach i = {1,2,3} in {
-  defvar shxadd = !cast<Instruction>("SH"#i#"ADD");
-  def : Pat<(i32 (add_like_non_imm12 (shl GPR:$rs1, (i64 i)), GPR:$rs2)),
-            (shxadd GPR:$rs1, GPR:$rs2)>;
-  def : Pat<(i32 (riscv_shl_add GPR:$rs1, (i32 i), GPR:$rs2)),
-            (shxadd GPR:$rs1, GPR:$rs2)>;
-}
-}
-
-let Predicates = [HasStdExtZbs, IsRV64] in {
-def : Pat<(i32 (and (not (shiftop<shl> 1, (i64 GPR:$rs2))), GPR:$rs1)),
-          (BCLR GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (and (rotl -2, (i64 GPR:$rs2)), GPR:$rs1)),
-          (BCLR GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (or (shiftop<shl> 1, (i64 GPR:$rs2)), GPR:$rs1)),
-          (BSET GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (xor (shiftop<shl> 1, (i64 GPR:$rs2)), GPR:$rs1)),
-          (BINV GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i32 (and (shiftop<srl> GPR:$rs1, (i64 GPR:$rs2)), 1)),
-          (BEXT GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i64 (and (anyext (i32 (shiftop<srl> GPR:$rs1, (i64 GPR:$rs2)))), 1)),
-          (BEXT GPR:$rs1, GPR:$rs2)>;
-
-def : Pat<(i32 (shiftop<shl> 1, (i64 GPR:$rs2))),
-          (BSET (XLenVT X0), GPR:$rs2)>;
-def : Pat<(i32 (not (shiftop<shl> -1, (i64 GPR:$rs2)))),
-          (ADDI (i32 (BSET (XLenVT X0), GPR:$rs2)), -1)>;
-
-def : Pat<(i32 (and (srl GPR:$rs1, uimm5:$shamt), (i32 1))),
-          (BEXTI GPR:$rs1, uimm5:$shamt)>;
-
-def : Pat<(i32 (and GPR:$rs1, BCLRMaski32:$mask)),
-          (BCLRI GPR:$rs1, (i64 (BCLRXForm $mask)))>;
-def : Pat<(i32 (or GPR:$rs1, SingleBitSetMaski32:$mask)),
-          (BSETI GPR:$rs1, (i64 (SingleBitSetMaskToIndex $mask)))>;
-def : Pat<(i32 (xor GPR:$rs1, SingleBitSetMaski32:$mask)),
-          (BINVI GPR:$rs1, (i64 (SingleBitSetMaskToIndex $mask)))>;
-} // Predicates = [HasStdExtZbs, IsRV64]
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index c759fdcbd0a1ea7..8fa5106cef32e94 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1091,16 +1091,30 @@ static bool generateGroupInst(const SPIRV::IncomingCall *Call,
 
   Register Arg0;
   if (GroupBuiltin->HasBoolArg) {
-    Register ConstRegister = Call->Arguments[0];
-    auto ArgInstruction = getDefInstrMaybeConstant(ConstRegister, MRI);
-    (void)ArgInstruction;
-    // TODO: support non-constant bool values.
-    assert(ArgInstruction->getOpcode() == TargetOpcode::G_CONSTANT &&
-           "Only constant bool value args are supported");
-    if (GR->getSPIRVTypeForVReg(Call->Arguments[0])->getOpcode() !=
-        SPIRV::OpTypeBool)
-      Arg0 = GR->buildConstantInt(getIConstVal(ConstRegister, MRI), MIRBuilder,
-                                  GR->getOrCreateSPIRVBoolType(MIRBuilder));
+    SPIRVType *BoolType = GR->getOrCreateSPIRVBoolType(MIRBuilder);
+    Register BoolReg = Call->Arguments[0];
+    SPIRVType *BoolRegType = GR->getSPIRVTypeForVReg(BoolReg);
+    if (!BoolRegType)
+      report_fatal_error("Can't find a register's type definition");
+    MachineInstr *ArgInstruction = getDefInstrMaybeConstant(BoolReg, MRI);
+    if (ArgInstruction->getOpcode() == TargetOpcode::G_CONSTANT) {
+      if (BoolRegType->getOpcode() != SPIRV::OpTypeBool)
+        Arg0 = GR->buildConstantInt(getIConstVal(BoolReg, MRI), MIRBuilder,
+                                    BoolType);
+    } else {
+      if (BoolRegType->getOpcode() == SPIRV::OpTypeInt) {
+        Arg0 = MRI->createGenericVirtualRegister(LLT::scalar(1));
+        MRI->setRegClass(Arg0, &SPIRV::IDRegClass);
+        GR->assignSPIRVTypeToVReg(BoolType, Arg0, MIRBuilder.getMF());
+        MIRBuilder.buildICmp(CmpInst::ICMP_NE, Arg0, BoolReg,
+                             GR->buildConstantInt(0, MIRBuilder, BoolRegType));
+        insertAssignInstr(Arg0, nullptr, BoolType, GR, MIRBuilder,
+                          MIRBuilder.getMF().getRegInfo());
+      } else if (BoolRegType->getOpcode() != SPIRV::OpTypeBool) {
+        report_fatal_error("Expect a boolean argument");
+      }
+      // if BoolReg is a boolean register, we don't need to do anything
+    }
   }
 
   Register GroupResultRegister = Call->ReturnRegister;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index c55235a04a607f7..1a8341ec48aaf5c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -175,6 +175,9 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectFmix(Register ResVReg, const SPIRVType *ResType,
                   MachineInstr &I) const;
 
+  bool selectLength(Register ResVReg, const SPIRVType *ResType,
+                    MachineInstr &I) const;
+
   bool selectFrac(Register ResVReg, const SPIRVType *ResType,
                   MachineInstr &I) const;
 
@@ -1372,6 +1375,23 @@ bool SPIRVInstructionSelector::selectFmix(Register ResVReg,
       .constrainAllUses(TII, TRI, RBI);
 }
 
+bool SPIRVInstructionSelector::selectLength(Register ResVReg,
+                                            const SPIRVType *ResType,
+                                            MachineInstr &I) const {
+
+  assert(I.getNumOperands() == 3);
+  assert(I.getOperand(2).isReg());
+  MachineBasicBlock &BB = *I.getParent();
+
+  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
+      .addDef(ResVReg)
+      .addUse(GR.getSPIRVTypeID(ResType))
+      .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
+      .addImm(GL::Length)
+      .addUse(I.getOperand(2).getReg())
+      .constrainAllUses(TII, TRI, RBI);
+}
+
 bool SPIRVInstructionSelector::selectFrac(Register ResVReg,
                                            const SPIRVType *ResType,
                                            MachineInstr &I) const {
@@ -2118,6 +2138,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
     return selectAny(ResVReg, ResType, I);
   case Intrinsic::spv_lerp:
     return selectFmix(ResVReg, ResType, I);
+  case Intrinsic::spv_length:
+    return selectLength(ResVReg, ResType, I);
   case Intrinsic::spv_frac:
     return selectFrac(ResVReg, ResType, I);
   case Intrinsic::spv_rsqrt:
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 7e2e97d387a83fb..39ba7ea777909c0 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3591,10 +3591,10 @@ static bool isX87Reg(unsigned Reg) {
 
 /// check if the instruction is X87 instruction
 bool X86::isX87Instruction(MachineInstr &MI) {
-  // Call defs X87 register, so we special case it here because
+  // Call and inlineasm defs X87 register, so we special case it here because
   // otherwise calls are incorrectly flagged as x87 instructions
   // as a result.
-  if (MI.isCall())
+  if (MI.isCall() || MI.isInlineAsm())
     return false;
   for (const MachineOperand &MO : MI.operands()) {
     if (!MO.isReg())
diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index c9de9c964bba0ae..66b68d5cd457fb9 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -242,16 +242,9 @@ class CallsiteContextGraph {
     // recursion.
     bool Recursive = false;
 
-    // The corresponding allocation or interior call. This is the primary call
-    // for which we have created this node.
+    // The corresponding allocation or interior call.
     CallInfo Call;
 
-    // List of other calls that can be treated the same as the primary call
-    // through cloning. I.e. located in the same function and have the same
-    // (possibly pruned) stack ids. They will be updated the same way as the
-    // primary call when assigning to function clones.
-    std::vector<CallInfo> MatchingCalls;
-
     // For alloc nodes this is a unique id assigned when constructed, and for
     // callsite stack nodes it is the original stack id when the node is
     // constructed from the memprof MIB metadata on the alloc nodes. Note that
@@ -464,9 +457,6 @@ class CallsiteContextGraph {
   /// iteration.
   MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
 
-  /// Records the function each call is located in.
-  DenseMap<CallInfo, const FuncTy *> CallToFunc;
-
   /// Map from callsite node to the enclosing caller function.
   std::map<const ContextNode *, const FuncTy *> NodeToCallingFunc;
 
@@ -484,8 +474,7 @@ class CallsiteContextGraph {
   /// StackIdToMatchingCalls map.
   void assignStackNodesPostOrder(
       ContextNode *Node, DenseSet<const ContextNode *> &Visited,
-      DenseMap<uint64_t, std::vector<CallContextInfo>> &StackIdToMatchingCalls,
-      DenseMap<CallInfo, CallInfo> &CallToMatchingCall);
+      DenseMap<uint64_t, std::vector<CallContextInfo>> &StackIdToMatchingCalls);
 
   /// Duplicates the given set of context ids, updating the provided
   /// map from each original id with the newly generated context ids,
@@ -1241,11 +1230,10 @@ static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
 
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
-    assignStackNodesPostOrder(
-        ContextNode *Node, DenseSet<const ContextNode *> &Visited,
-        DenseMap<uint64_t, std::vector<CallContextInfo>>
-            &StackIdToMatchingCalls,
-        DenseMap<CallInfo, CallInfo> &CallToMatchingCall) {
+    assignStackNodesPostOrder(ContextNode *Node,
+                              DenseSet<const ContextNode *> &Visited,
+                              DenseMap<uint64_t, std::vector<CallContextInfo>>
+                                  &StackIdToMatchingCalls) {
   auto Inserted = Visited.insert(Node);
   if (!Inserted.second)
     return;
@@ -1258,8 +1246,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     // Skip any that have been removed during the recursion.
     if (!Edge)
       continue;
-    assignStackNodesPostOrder(Edge->Caller, Visited, StackIdToMatchingCalls,
-                              CallToMatchingCall);
+    assignStackNodesPostOrder(Edge->Caller, Visited, StackIdToMatchingCalls);
   }
 
   // If this node's stack id is in the map, update the graph to contain new
@@ -1302,19 +1289,8 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
     auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
     // Skip any for which we didn't assign any ids, these don't get a node in
     // the graph.
-    if (SavedContextIds.empty()) {
-      // If this call has a matching call (located in the same function and
-      // having the same stack ids), simply add it to the context node created
-      // for its matching call earlier. These can be treated the same through
-      // cloning and get updated at the same time.
-      if (!CallToMatchingCall.contains(Call))
-        continue;
-      auto MatchingCall = CallToMatchingCall[Call];
-      assert(NonAllocationCallToContextNodeMap.contains(MatchingCall));
-      NonAllocationCallToContextNodeMap[MatchingCall]->MatchingCalls.push_back(
-          Call);
+    if (SavedContextIds.empty())
       continue;
-    }
 
     assert(LastId == Ids.back());
 
@@ -1446,10 +1422,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
   // there is more than one call with the same stack ids. Their (possibly newly
   // duplicated) context ids are saved in the StackIdToMatchingCalls map.
   DenseMap<uint32_t, DenseSet<uint32_t>> OldToNewContextIds;
-  // Save a map from each call to any that are found to match it. I.e. located
-  // in the same function and have the same (possibly pruned) stack ids. We use
-  // this to avoid creating extra graph nodes as they can be treated the same.
-  DenseMap<CallInfo, CallInfo> CallToMatchingCall;
   for (auto &It : StackIdToMatchingCalls) {
     auto &Calls = It.getSecond();
     // Skip single calls with a single stack id. These don't need a new node.
@@ -1488,13 +1460,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
     DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
     assert(!LastNodeContextIds.empty());
 
-    // Map from function to the first call from the below list (with matching
-    // stack ids) found in that function. Note that calls from different
-    // functions can have the same stack ids because this is the list of stack
-    // ids that had (possibly pruned) nodes after building the graph from the
-    // allocation MIBs.
-    DenseMap<const FuncTy *, CallInfo> FuncToCallMap;
-
     for (unsigned I = 0; I < Calls.size(); I++) {
       auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
       assert(SavedContextIds.empty());
@@ -1568,18 +1533,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
           continue;
       }
 
-      const FuncTy *CallFunc = CallToFunc[Call];
-
-      // If the prior call had the same stack ids this map would not be empty.
-      // Check if we already have a call that "matches" because it is located
-      // in the same function.
-      if (FuncToCallMap.contains(CallFunc)) {
-        // Record the matching call found for this call, and skip it. We
-        // will subsequently combine it into the same node.
-        CallToMatchingCall[Call] = FuncToCallMap[CallFunc];
-        continue;
-      }
-
       // Check if the next set of stack ids is the same (since the Calls vector
       // of tuples is sorted by the stack ids we can just look at the next one).
       bool DuplicateContextIds = false;
@@ -1609,14 +1562,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
         set_subtract(LastNodeContextIds, StackSequenceContextIds);
         if (LastNodeContextIds.empty())
           break;
-        // No longer possibly in a sequence of calls with duplicate stack ids,
-        // clear the map.
-        FuncToCallMap.clear();
-      } else
-        // Record the call with its function, so we can locate it the next time
-        // we find a call from this function when processing the calls with the
-        // same stack ids.
-        FuncToCallMap[CallFunc] = Call;
+      }
     }
   }
 
@@ -1633,8 +1579,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
   // associated context ids over to the new nodes.
   DenseSet<const ContextNode *> Visited;
   for (auto &Entry : AllocationCallToContextNodeMap)
-    assignStackNodesPostOrder(Entry.second, Visited, StackIdToMatchingCalls,
-                              CallToMatchingCall);
+    assignStackNodesPostOrder(Entry.second, Visited, StackIdToMatchingCalls);
   if (VerifyCCG)
     check();
 }
@@ -1734,7 +1679,6 @@ ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
           continue;
         if (auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof)) {
           CallsWithMetadata.push_back(&I);
-          CallToFunc[&I] = &F;
           auto *AllocNode = addAllocNode(&I, &F);
           auto *CallsiteMD = I.getMetadata(LLVMContext::MD_callsite);
           assert(CallsiteMD);
@@ -1756,10 +1700,8 @@ ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
           I.setMetadata(LLVMContext::MD_callsite, nullptr);
         }
         // For callsite metadata, add to list for this function for later use.
-        else if (I.getMetadata(LLVMContext::MD_callsite)) {
+        else if (I.getMetadata(LLVMContext::MD_callsite))
           CallsWithMetadata.push_back(&I);
-          CallToFunc[&I] = &F;
-        }
       }
     }
     if (!CallsWithMetadata.empty())
@@ -1814,10 +1756,8 @@ IndexCallsiteContextGraph::IndexCallsiteContextGraph(
           // correlate properly in applyImport in the backends.
           if (AN.MIBs.empty())
             continue;
-          IndexCall AllocCall(&AN);
-          CallsWithMetadata.push_back(AllocCall);
-          CallToFunc[AllocCall] = FS;
-          auto *AllocNode = addAllocNode(AllocCall, FS);
+          CallsWithMetadata.push_back({&AN});
+          auto *AllocNode = addAllocNode({&AN}, FS);
           // Pass an empty CallStack to the CallsiteContext (second)
           // parameter, since for ThinLTO we already collapsed out the inlined
           // stack ids on the allocation call during ModuleSummaryAnalysis.
@@ -1848,11 +1788,8 @@ IndexCallsiteContextGraph::IndexCallsiteContextGraph(
       }
       // For callsite metadata, add to list for this function for later use.
       if (!FS->callsites().empty())
-        for (auto &SN : FS->mutableCallsites()) {
-          IndexCall StackNodeCall(&SN);
-          CallsWithMetadata.push_back(StackNodeCall);
-          CallToFunc[StackNodeCall] = FS;
-        }
+        for (auto &SN : FS->mutableCallsites())
+          CallsWithMetadata.push_back({&SN});
 
       if (!CallsWithMetadata.empty())
         FuncToCallsWithMetadata[FS] = CallsWithMetadata;
@@ -2288,14 +2225,6 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::print(
   if (Recursive)
     OS << " (recursive)";
   OS << "\n";
-  if (!MatchingCalls.empty()) {
-    OS << "\tMatchingCalls:\n";
-    for (auto &MatchingCall : MatchingCalls) {
-      OS << "\t";
-      MatchingCall.print(OS);
-      OS << "\n";
-    }
-  }
   OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n";
   OS << "\tContextIds:";
   // Make a copy of the computed context ids that we can sort for stability.
@@ -2549,7 +2478,6 @@ CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::moveEdgeToNewCalleeClone(
       std::make_unique<ContextNode>(Node->IsAllocation, Node->Call));
   ContextNode *Clone = NodeOwner.back().get();
   Node->addClone(Clone);
-  Clone->MatchingCalls = Node->MatchingCalls;
   assert(NodeToCallingFunc.count(Node));
   NodeToCallingFunc[Clone] = NodeToCallingFunc[Node];
   moveEdgeToExistingCalleeClone(Edge, Clone, CallerEdgeI, /*NewClone=*/true,
@@ -3093,14 +3021,6 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
         if (CallMap.count(Call))
           CallClone = CallMap[Call];
         CallsiteClone->setCall(CallClone);
-        // Need to do the same for all matching calls.
-        for (auto &MatchingCall : Node->MatchingCalls) {
-          CallInfo CallClone(MatchingCall);
-          if (CallMap.count(MatchingCall))
-            CallClone = CallMap[MatchingCall];
-          // Updates the call in the list.
-          MatchingCall = CallClone;
-        }
       };
 
       // Keep track of the clones of callsite Node that need to be assigned to
@@ -3267,16 +3187,6 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
               CallInfo NewCall(CallMap[OrigCall]);
               assert(NewCall);
               NewClone->setCall(NewCall);
-              // Need to do the same for all matching calls.
-              for (auto &MatchingCall : NewClone->MatchingCalls) {
-                CallInfo OrigMatchingCall(MatchingCall);
-                OrigMatchingCall.setCloneNo(0);
-                assert(CallMap.count(OrigMatchingCall));
-                CallInfo NewCall(CallMap[OrigMatchingCall]);
-                assert(NewCall);
-                // Updates the call in the list.
-                MatchingCall = NewCall;
-              }
             }
           }
           // Fall through to handling below to perform the recording of the
@@ -3463,7 +3373,6 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
 
     if (Node->IsAllocation) {
       updateAllocationCall(Node->Call, allocTypeToUse(Node->AllocTypes));
-      assert(Node->MatchingCalls.empty());
       return;
     }
 
@@ -3472,9 +3381,6 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
 
     auto CalleeFunc = CallsiteToCalleeFuncCloneMap[Node];
     updateCall(Node->Call, CalleeFunc);
-    // Update all the matching calls as well.
-    for (auto &Call : Node->MatchingCalls)
-      updateCall(Call, CalleeFunc);
   };
 
   // Performs DFS traversal starting from allocation nodes to update calls to
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index c31173879af1e65..84ccf06d16d5e81 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -875,7 +875,7 @@ void ConstraintInfo::transferToOtherSystem(
       addFact(CmpInst::ICMP_ULT, A, B, NumIn, NumOut, DFSInStack);
     break;
   case CmpInst::ICMP_SGT: {
-    if (doesHold(CmpInst::ICMP_SGE, B, ConstantInt::get(B->getType(), -1)))
+    if (doesHold(CmpInst::ICMP_SGE, B, Constant::getAllOnesValue(B->getType())))
       addFact(CmpInst::ICMP_UGE, A, ConstantInt::get(B->getType(), 0), NumIn,
               NumOut, DFSInStack);
     if (IsKnownNonNegative(B))
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 41e4b0fcd9b0db2..5547116133e8da2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8545,13 +8545,18 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
 static void addUsersInExitBlock(
     Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan,
     const MapVector<PHINode *, InductionDescriptor> &Inductions) {
-  BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock();
-  BasicBlock *ExitingBB = OrigLoop->getExitingBlock();
-  // Only handle single-exit loops with unique exit blocks for now.
-  if (!ExitBB || !ExitBB->getSinglePredecessor() || !ExitingBB)
+  auto MiddleVPBB =
+      cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSingleSuccessor());
+  // No edge from the middle block to the unique exit block has been inserted
+  // and there is nothing to fix from vector loop; phis should have incoming
+  // from scalar loop only.
+  if (MiddleVPBB->getNumSuccessors() != 2)
     return;
 
   // Introduce VPUsers modeling the exit values.
+  BasicBlock *ExitBB =
+      cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0])->getIRBasicBlock();
+  BasicBlock *ExitingBB = OrigLoop->getExitingBlock();
   for (PHINode &ExitPhi : ExitBB->phis()) {
     Value *IncomingValue =
         ExitPhi.getIncomingValueForBlock(ExitingBB);
@@ -8777,13 +8782,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
   // After here, VPBB should not be used.
   VPBB = nullptr;
 
-  if (CM.requiresScalarEpilogue(Range)) {
-    // No edge from the middle block to the unique exit block has been inserted
-    // and there is nothing to fix from vector loop; phis should have incoming
-    // from scalar loop only.
-  } else
-    addUsersInExitBlock(OrigLoop, RecipeBuilder, *Plan,
-                        Legal->getInductionVars());
+  addUsersInExitBlock(OrigLoop, RecipeBuilder, *Plan,
+                      Legal->getInductionVars());
 
   assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
          !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c05e4b822d2c4eb..f30a76af8327904 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -10135,13 +10135,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       }
       SmallVector<int> Mask;
       E->buildAltOpShuffleMask(
-          [E](Instruction *I) {
+          [&](Instruction *I) {
             assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
-            return I->getOpcode() == E->getAltOpcode();
+            return isAlternateInstruction(I, E->getMainOp(), E->getAltOp(),
+                                          *TLI);
           },
           Mask);
       VecCost += ::getShuffleCost(TTIRef, TargetTransformInfo::SK_PermuteTwoSrc,
-                                  FinalVecTy, Mask);
+                                  FinalVecTy, Mask, CostKind);
       // Patterns like [fadd,fsub] can be combined into a single instruction
       // in x86. Reordering them into [fsub,fadd] blocks this pattern. So we
       // need to take into account their order when looking for the most used
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll
index 64306ac28cf2752..b313842ad5e1a98 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count-implied-addrec.ll
@@ -239,12 +239,6 @@ define void @neg_rhs_wrong_range(i16 %n.raw) mustprogress {
 ; CHECK-NEXT:  Loop %for.body: Unpredictable backedge-taken count.
 ; CHECK-NEXT:  Loop %for.body: Unpredictable constant max backedge-taken count.
 ; CHECK-NEXT:  Loop %for.body: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT:  Loop %for.body: Predicated backedge-taken count is ((-1 + (2 umax (-1 + (zext i8 (trunc i16 %n.raw to i8) to i16))<nsw>)) /u 2)
-; CHECK-NEXT:   Predicates:
-; CHECK-NEXT:      {2,+,2}<nw><%for.body> Added Flags: <nusw>
-; CHECK-NEXT:  Loop %for.body: Predicated symbolic max backedge-taken count is ((-1 + (2 umax (-1 + (zext i8 (trunc i16 %n.raw to i8) to i16))<nsw>)) /u 2)
-; CHECK-NEXT:   Predicates:
-; CHECK-NEXT:      {2,+,2}<nw><%for.body> Added Flags: <nusw>
 ;
 entry:
   %n.and = and i16 %n.raw, 255
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-scalable-stride.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-scalable-stride.ll
index 6f26a8a64e7188f..30a095fd144fa4e 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count-scalable-stride.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count-scalable-stride.ll
@@ -377,7 +377,7 @@ define void @vscale_slt_noflags(ptr nocapture %A, i32 %n) mustprogress vscale_ra
 ; CHECK-NEXT:    %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
 ; CHECK-NEXT:    --> {%A,+,(4 * vscale)<nuw><nsw>}<%for.body> U: full-set S: full-set Exits: ((4 * vscale * ((-1 + %n) /u vscale)) + %A) LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %add = add i32 %i.05, %vscale
-; CHECK-NEXT:    --> {vscale,+,vscale}<nw><%for.body> U: full-set S: full-set Exits: (vscale * (1 + ((-1 + %n) /u vscale))<nuw>) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    --> {vscale,+,vscale}<nuw><nsw><%for.body> U: [2,-2147483648) S: [2,-2147483648) Exits: (vscale * (1 + ((-1 + %n) /u vscale))<nuw>) LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:  Determining loop execution counts for: @vscale_slt_noflags
 ; CHECK-NEXT:  Loop %for.body: backedge-taken count is ((-1 + %n) /u vscale)
 ; CHECK-NEXT:  Loop %for.body: constant max backedge-taken count is i32 1073741822
@@ -415,7 +415,7 @@ define void @vscalex4_ult_noflags(ptr nocapture %A, i32 %n) mustprogress vscale_
 ; CHECK-NEXT:    %arrayidx = getelementptr inbounds i32, ptr %A, i32 %i.05
 ; CHECK-NEXT:    --> {%A,+,(16 * vscale)<nuw><nsw>}<%for.body> U: full-set S: full-set Exits: ((16 * vscale * ((-1 + %n) /u (4 * vscale)<nuw><nsw>)) + %A) LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %add = add i32 %i.05, %VF
-; CHECK-NEXT:    --> {(4 * vscale)<nuw><nsw>,+,(4 * vscale)<nuw><nsw>}<nw><%for.body> U: [0,-3) S: [-2147483648,2147483645) Exits: (vscale * (4 + (4 * ((-1 + %n) /u (4 * vscale)<nuw><nsw>))<nuw><nsw>)<nuw>) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    --> {(4 * vscale)<nuw><nsw>,+,(4 * vscale)<nuw><nsw>}<nuw><%for.body> U: [8,-3) S: [-2147483648,2147483645) Exits: (vscale * (4 + (4 * ((-1 + %n) /u (4 * vscale)<nuw><nsw>))<nuw><nsw>)<nuw>) LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:  Determining loop execution counts for: @vscalex4_ult_noflags
 ; CHECK-NEXT:  Loop %for.body: backedge-taken count is ((-1 + %n) /u (4 * vscale)<nuw><nsw>)
 ; CHECK-NEXT:  Loop %for.body: constant max backedge-taken count is i32 536870910
diff --git a/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll b/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
new file mode 100644
index 000000000000000..b9eda0c1cd3bb6e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/print-pipeline-passes.ll
@@ -0,0 +1,12 @@
+; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto<O0>" -print-pipeline-passes %s -o - | FileCheck --check-prefix=O0 %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto<O1>" -print-pipeline-passes %s -o - | FileCheck %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto<O2>" -print-pipeline-passes %s -o - | FileCheck %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -passes="lto<O3>" -print-pipeline-passes %s -o - | FileCheck %s
+
+; CHECK: amdgpu-attributor
+; O0-NOT: amdgpu-attributor
+
+define amdgpu_kernel void @kernel() {
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-true16.mir b/llvm/test/CodeGen/AMDGPU/shrink-true16.mir
new file mode 100644
index 000000000000000..4b2e3951ce4c13f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/shrink-true16.mir
@@ -0,0 +1,28 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=si-shrink-instructions -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX1100 %s
+
+---
+name: 16bit_lo128_shrink
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr127
+    ; GFX1100-LABEL: name: 16bit_lo128_shrink
+    ; GFX1100: liveins: $vgpr127
+    ; GFX1100-NEXT: {{  $}}
+    ; GFX1100-NEXT: V_CMP_EQ_U16_t16_e32 0, $vgpr127, implicit-def $vcc, implicit $exec, implicit $exec
+    $vcc_lo = V_CMP_EQ_U16_t16_e64 0, $vgpr127, implicit-def $vcc, implicit $exec
+...
+
+---
+name: 16bit_lo128_no_shrink
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr128
+    ; GFX1100-LABEL: name: 16bit_lo128_no_shrink
+    ; GFX1100: liveins: $vgpr128
+    ; GFX1100-NEXT: {{  $}}
+    ; GFX1100-NEXT: $vcc_lo = V_CMP_EQ_U16_t16_e64 0, $vgpr128, implicit-def $vcc_lo, implicit $exec
+    $vcc_lo = V_CMP_EQ_U16_t16_e64 0, $vgpr128, implicit-def $vcc, implicit $exec
+...
diff --git a/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll
index 254479e5f94cbda..b9a8e3e80567e09 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.0.ll
@@ -1,9 +1,14 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel6.0-vertex"
 
 ; CHECK: !dx.version = !{![[DXVER:[0-9]+]]}
 ; CHECK: ![[DXVER]] = !{i32 1, i32 0}
 
+; ANALYSIS: Shader Model Version : 6.0
+; ANALYSIS: DXIL Version : 1.0
+; ANALYSIS: Shader Stage : vertex
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll
index efeb5a1b24862e5..fdd21d627829b92 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/dxilVer-1.8.ll
@@ -1,9 +1,14 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel6.8-compute"
 
 ; CHECK: !dx.version = !{![[DXVER:[0-9]+]]}
 ; CHECK: ![[DXVER]] = !{i32 1, i32 8}
 
+; ANALYSIS: Shader Model Version : 6.8
+; ANALYSIS: DXIL Version : 1.8
+; ANALYSIS: Shader Stage : compute
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-as.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-as.ll
index fe3361c781ce426..d2625fc8b96a9da 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-as.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-as.ll
@@ -1,9 +1,14 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel6-amplification"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"as", i32 6, i32 0}
 
+; ANALYSIS: Shader Model Version : 6
+; ANALYSIS: DXIL Version : 1.0
+; ANALYSIS: Shader Stage : amplification
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll
index a85dc43ac2f6c2c..8e8f9d77539eabf 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs-val-ver-0.0.ll
@@ -1,4 +1,5 @@
-; RUN: opt -S -dxil-prepare  %s | FileCheck %s 
+; RUN: opt -S -dxil-prepare  %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 
 target triple = "dxil-pc-shadermodel6.6-compute"
 
@@ -8,9 +9,13 @@ entry:
 }
 
 ; Make sure experimental attribute is left when validation version is 0.0.
-; CHECK:attributes #0 = { noinline nounwind "exp-shader"="cs" } 
+; CHECK:attributes #0 = { noinline nounwind "exp-shader"="cs" }
 attributes #0 = { noinline nounwind "exp-shader"="cs" "hlsl.numthreads"="1,2,1" "hlsl.shader"="compute" }
 
 !dx.valver = !{!0}
 
 !0 = !{i32 0, i32 0}
+
+; ANALYSIS: Shader Model Version : 6.6
+; ANALYSIS: DXIL Version : 1.6
+; ANALYSIS: Shader Stage : compute
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll
index 343f190d994f0db..24eb0d608d8bb64 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-cs.ll
@@ -1,11 +1,16 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
 ; RUN: opt -S -dxil-prepare  %s | FileCheck %s  --check-prefix=REMOVE_EXTRA_ATTRIBUTE
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 
 target triple = "dxil-pc-shadermodel6.6-compute"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"cs", i32 6, i32 6}
 
+; ANALYSIS: Shader Model Version : 6.6
+; ANALYSIS: DXIL Version : 1.6
+; ANALYSIS: Shader Stage : compute
+
 define void @entry() #0 {
 entry:
   ret void
@@ -13,5 +18,5 @@ entry:
 
 ; Make sure extra attribute like hlsl.numthreads are removed.
 ; And experimental attribute is removed when validator version is not 0.0.
-; REMOVE_EXTRA_ATTRIBUTE:attributes #0 = { noinline nounwind } 
+; REMOVE_EXTRA_ATTRIBUTE:attributes #0 = { noinline nounwind }
 attributes #0 = { noinline nounwind "exp-shader"="cs" "hlsl.numthreads"="1,2,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-gs.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-gs.ll
index a0a1b7c2ff30308..5c28c9305f01bfd 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-gs.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-gs.ll
@@ -1,9 +1,14 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel6.6-geometry"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"gs", i32 6, i32 6}
 
+; ANALYSIS: Shader Model Version : 6.6
+; ANALYSIS: DXIL Version : 1.6
+; ANALYSIS: Shader Stage : geometry
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-hs.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-hs.ll
index 6b1fa46c2c137f0..e60023d1b3a5fb4 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-hs.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-hs.ll
@@ -1,9 +1,14 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel6.6-hull"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"hs", i32 6, i32 6}
 
+; ANALYSIS: Shader Model Version : 6.6
+; ANALYSIS: DXIL Version : 1.6
+; ANALYSIS: Shader Stage : hull
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-lib.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-lib.ll
index 3644cf21bcaebfd..7f0bea95c048225 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-lib.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-lib.ll
@@ -1,5 +1,10 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel6.3-library"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"lib", i32 6, i32 3}
+
+; ANALYSIS: Shader Model Version : 6.3
+; ANALYSIS: DXIL Version : 1.3
+; ANALYSIS: Shader Stage : library
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ms.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ms.ll
index 766e8e2e30b5a55..dd033b9a9722b94 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ms.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ms.ll
@@ -1,9 +1,14 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel6.6-mesh"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"ms", i32 6, i32 6}
 
+; ANALYSIS: Shader Model Version : 6.6
+; ANALYSIS: DXIL Version : 1.6
+; ANALYSIS: Shader Stage : mesh
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ps.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ps.ll
index 46e8f3bcfa8565c..47da321df3e42e5 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ps.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-ps.ll
@@ -1,9 +1,15 @@
+
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel5.0-pixel"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"ps", i32 5, i32 0}
 
+; ANALYSIS: Shader Model Version : 5.0
+; ANALYSIS: DXIL Version : 1.0
+; ANALYSIS: Shader Stage : pixel
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-vs.ll b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-vs.ll
index 7a0cfdf8162661e..dcc68586dd4b9f2 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/shaderModel-vs.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/shaderModel-vs.ll
@@ -1,9 +1,14 @@
 ; RUN: opt -S -dxil-metadata-emit %s | FileCheck %s
+; RUN: opt -S -passes="print<dxil-metadata>" -disable-output %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 target triple = "dxil-pc-shadermodel-vertex"
 
 ; CHECK: !dx.shaderModel = !{![[SM:[0-9]+]]}
 ; CHECK: ![[SM]] = !{!"vs", i32 0, i32 0}
 
+; ANALYSIS: Shader Model Version : 0
+; ANALYSIS: DXIL Version : 1.0
+; ANALYSIS: Shader Stage : vertex
+
 define void @entry() #0 {
 entry:
   ret void
diff --git a/llvm/test/CodeGen/Mips/bittest.ll b/llvm/test/CodeGen/Mips/bittest.ll
index 167140c83b06697..062c8002490f52b 100644
--- a/llvm/test/CodeGen/Mips/bittest.ll
+++ b/llvm/test/CodeGen/Mips/bittest.ll
@@ -53,23 +53,22 @@ define signext i32 @bittest_10_i32(i32 signext %a) nounwind {
 ; MIPS64:       # %bb.0:
 ; MIPS64-NEXT:    andi $1, $4, 1024
 ; MIPS64-NEXT:    sltiu $1, $1, 1
-; MIPS64-NEXT:    dsll $1, $1, 32
 ; MIPS64-NEXT:    jr $ra
-; MIPS64-NEXT:    dsrl $2, $1, 32
+; MIPS64-NEXT:    sll $2, $1, 0
 ;
 ; MIPS64R2-LABEL: bittest_10_i32:
 ; MIPS64R2:       # %bb.0:
 ; MIPS64R2-NEXT:    andi $1, $4, 1024
 ; MIPS64R2-NEXT:    sltiu $1, $1, 1
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    dext $2, $1, 0, 32
+; MIPS64R2-NEXT:    sll $2, $1, 0
 ;
 ; MIPS64R6-LABEL: bittest_10_i32:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    andi $1, $4, 1024
 ; MIPS64R6-NEXT:    sltiu $1, $1, 1
 ; MIPS64R6-NEXT:    jr $ra
-; MIPS64R6-NEXT:    dext $2, $1, 0, 32
+; MIPS64R6-NEXT:    sll $2, $1, 0
 ;
 ; MM32R3-LABEL: bittest_10_i32:
 ; MM32R3:       # %bb.0:
@@ -111,23 +110,22 @@ define signext i32 @bittest_15_i32(i32 signext %a) nounwind {
 ; MIPS64:       # %bb.0:
 ; MIPS64-NEXT:    andi $1, $4, 32768
 ; MIPS64-NEXT:    sltiu $1, $1, 1
-; MIPS64-NEXT:    dsll $1, $1, 32
 ; MIPS64-NEXT:    jr $ra
-; MIPS64-NEXT:    dsrl $2, $1, 32
+; MIPS64-NEXT:    sll $2, $1, 0
 ;
 ; MIPS64R2-LABEL: bittest_15_i32:
 ; MIPS64R2:       # %bb.0:
 ; MIPS64R2-NEXT:    andi $1, $4, 32768
 ; MIPS64R2-NEXT:    sltiu $1, $1, 1
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    dext $2, $1, 0, 32
+; MIPS64R2-NEXT:    sll $2, $1, 0
 ;
 ; MIPS64R6-LABEL: bittest_15_i32:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    andi $1, $4, 32768
 ; MIPS64R6-NEXT:    sltiu $1, $1, 1
 ; MIPS64R6-NEXT:    jr $ra
-; MIPS64R6-NEXT:    dext $2, $1, 0, 32
+; MIPS64R6-NEXT:    sll $2, $1, 0
 ;
 ; MM32R3-LABEL: bittest_15_i32:
 ; MM32R3:       # %bb.0:
@@ -171,23 +169,22 @@ define signext i32 @bittest_16_i32(i32 signext %a) nounwind {
 ; MIPS64-NEXT:    not $1, $4
 ; MIPS64-NEXT:    srl $1, $1, 16
 ; MIPS64-NEXT:    andi $1, $1, 1
-; MIPS64-NEXT:    dsll $1, $1, 32
 ; MIPS64-NEXT:    jr $ra
-; MIPS64-NEXT:    dsrl $2, $1, 32
+; MIPS64-NEXT:    sll $2, $1, 0
 ;
 ; MIPS64R2-LABEL: bittest_16_i32:
 ; MIPS64R2:       # %bb.0:
 ; MIPS64R2-NEXT:    not $1, $4
 ; MIPS64R2-NEXT:    ext $1, $1, 16, 1
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    dext $2, $1, 0, 32
+; MIPS64R2-NEXT:    sll $2, $1, 0
 ;
 ; MIPS64R6-LABEL: bittest_16_i32:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    not $1, $4
 ; MIPS64R6-NEXT:    ext $1, $1, 16, 1
 ; MIPS64R6-NEXT:    jr $ra
-; MIPS64R6-NEXT:    dext $2, $1, 0, 32
+; MIPS64R6-NEXT:    sll $2, $1, 0
 ;
 ; MM32R3-LABEL: bittest_16_i32:
 ; MM32R3:       # %bb.0:
@@ -228,24 +225,20 @@ define signext i32 @bittest_31_i32(i32 signext %a) nounwind {
 ; MIPS64-LABEL: bittest_31_i32:
 ; MIPS64:       # %bb.0:
 ; MIPS64-NEXT:    not $1, $4
-; MIPS64-NEXT:    srl $1, $1, 31
-; MIPS64-NEXT:    dsll $1, $1, 32
 ; MIPS64-NEXT:    jr $ra
-; MIPS64-NEXT:    dsrl $2, $1, 32
+; MIPS64-NEXT:    srl $2, $1, 31
 ;
 ; MIPS64R2-LABEL: bittest_31_i32:
 ; MIPS64R2:       # %bb.0:
 ; MIPS64R2-NEXT:    not $1, $4
-; MIPS64R2-NEXT:    srl $1, $1, 31
 ; MIPS64R2-NEXT:    jr $ra
-; MIPS64R2-NEXT:    dext $2, $1, 0, 32
+; MIPS64R2-NEXT:    srl $2, $1, 31
 ;
 ; MIPS64R6-LABEL: bittest_31_i32:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    not $1, $4
-; MIPS64R6-NEXT:    srl $1, $1, 31
 ; MIPS64R6-NEXT:    jr $ra
-; MIPS64R6-NEXT:    dext $2, $1, 0, 32
+; MIPS64R6-NEXT:    srl $2, $1, 31
 ;
 ; MM32R3-LABEL: bittest_31_i32:
 ; MM32R3:       # %bb.0:
diff --git a/llvm/test/CodeGen/NVPTX/extractelement.ll b/llvm/test/CodeGen/NVPTX/extractelement.ll
index 92d00f881cc261d..367c20749a9f361 100644
--- a/llvm/test/CodeGen/NVPTX/extractelement.ll
+++ b/llvm/test/CodeGen/NVPTX/extractelement.ll
@@ -1,12 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 -verify-machineinstrs | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_35 | %ptxas-verify %}
+target triple = "nvptx64-nvidia-cuda"
 
 
-; CHECK-LABEL: test_v2i8
-; CHECK-DAG:        ld.param.u16    [[A:%rs[0-9]+]], [test_v2i8_param_0];
-; CHECK-DAG:        cvt.s16.s8      [[E0:%rs[0-9]+]], [[A]];
-; CHECK-DAG:        shr.s16         [[E1:%rs[0-9]+]], [[A]], 8;
 define i16  @test_v2i8(i16 %a) {
+; CHECK-LABEL: test_v2i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u16 %rs1, [test_v2i8_param_0];
+; CHECK-NEXT:    cvt.s16.s8 %rs2, %rs1;
+; CHECK-NEXT:    shr.s16 %rs3, %rs1, 8;
+; CHECK-NEXT:    add.s16 %rs4, %rs2, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs4;
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; CHECK-NEXT:    ret;
   %v = bitcast i16 %a to <2 x i8>
   %r0 = extractelement <2 x i8> %v, i64 0
   %r1 = extractelement <2 x i8> %v, i64 1
@@ -16,17 +27,53 @@ define i16  @test_v2i8(i16 %a) {
   ret i16 %r01
 }
 
-; CHECK-LABEL: test_v4i8
-; CHECK:            ld.param.u32    [[R:%r[0-9]+]], [test_v4i8_param_0];
-; CHECK-DAG:        bfe.s32         [[R0:%r[0-9]+]], [[R]], 0, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E0:%rs[0-9]+]], [[R0]];
-; CHECK-DAG:        bfe.s32         [[R1:%r[0-9]+]], [[R]], 8, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E1:%rs[0-9]+]], [[R1]];
-; CHECK-DAG:        bfe.s32         [[R2:%r[0-9]+]], [[R]], 16, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E2:%rs[0-9]+]], [[R2]];
-; CHECK-DAG:        bfe.s32         [[R3:%r[0-9]+]], [[R]], 24, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E3:%rs[0-9]+]], [[R3]];
+define i1  @test_v2i8_load(ptr %a) {
+; CHECK-LABEL: test_v2i8_load(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<7>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_v2i8_load_param_0];
+; CHECK-NEXT:    ld.v2.u8 {%rs1, %rs2}, [%rd1];
+; CHECK-NEXT:    or.b16 %rs5, %rs1, %rs2;
+; CHECK-NEXT:    and.b16 %rs6, %rs5, 255;
+; CHECK-NEXT:    setp.eq.s16 %p1, %rs6, 0;
+; CHECK-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; CHECK-NEXT:    ret;
+  %v = load <2 x i8>, ptr %a, align 4
+  %r0 = extractelement <2 x i8> %v, i64 0
+  %r1 = extractelement <2 x i8> %v, i64 1
+  %icmp = icmp eq i8 %r0, 0
+  %icmp3 = icmp eq i8 %r1, 0
+  %select = select i1 %icmp, i1 %icmp3, i1 false
+  ret i1 %select
+}
 define i16  @test_v4i8(i32 %a) {
+; CHECK-LABEL: test_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<8>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [test_v4i8_param_0];
+; CHECK-NEXT:    bfe.s32 %r2, %r1, 0, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs1, %r2;
+; CHECK-NEXT:    bfe.s32 %r3, %r1, 8, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs2, %r3;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 16, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs3, %r4;
+; CHECK-NEXT:    bfe.s32 %r5, %r1, 24, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs4, %r5;
+; CHECK-NEXT:    add.s16 %rs5, %rs1, %rs2;
+; CHECK-NEXT:    add.s16 %rs6, %rs3, %rs4;
+; CHECK-NEXT:    add.s16 %rs7, %rs5, %rs6;
+; CHECK-NEXT:    cvt.u32.u16 %r6, %rs7;
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r6;
+; CHECK-NEXT:    ret;
   %v = bitcast i32 %a to <4 x i8>
   %r0 = extractelement <4 x i8> %v, i64 0
   %r1 = extractelement <4 x i8> %v, i64 1
@@ -42,16 +89,22 @@ define i16  @test_v4i8(i32 %a) {
   ret i16 %r
 }
 
-; CHECK-LABEL: test_v4i8_s32
-; CHECK:            ld.param.u32    [[R:%r[0-9]+]], [test_v4i8_s32_param_0];
-; CHECK-DAG:        bfe.s32         [[R0:%r[0-9]+]], [[R]], 0, 8;
-; CHECK-DAG:        bfe.s32         [[R1:%r[0-9]+]], [[R]], 8, 8;
-; CHECK-DAG:        bfe.s32         [[R2:%r[0-9]+]], [[R]], 16, 8;
-; CHECK-DAG:        bfe.s32         [[R3:%r[0-9]+]], [[R]], 24, 8;
-; CHECK-DAG:        add.s32         [[R01:%r[0-9]+]], [[R0]], [[R1]]
-; CHECK-DAG:        add.s32         [[R23:%r[0-9]+]], [[R2]], [[R3]]
-; CHECK-DAG:        add.s32         [[R0123:%r[0-9]+]], [[R01]], [[R23]]
 define i32  @test_v4i8_s32(i32 %a) {
+; CHECK-LABEL: test_v4i8_s32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [test_v4i8_s32_param_0];
+; CHECK-NEXT:    bfe.s32 %r2, %r1, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r3, %r1, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r5, %r1, 24, 8;
+; CHECK-NEXT:    add.s32 %r6, %r2, %r3;
+; CHECK-NEXT:    add.s32 %r7, %r4, %r5;
+; CHECK-NEXT:    add.s32 %r8, %r6, %r7;
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r8;
+; CHECK-NEXT:    ret;
   %v = bitcast i32 %a to <4 x i8>
   %r0 = extractelement <4 x i8> %v, i64 0
   %r1 = extractelement <4 x i8> %v, i64 1
@@ -67,16 +120,22 @@ define i32  @test_v4i8_s32(i32 %a) {
   ret i32 %r
 }
 
-; CHECK-LABEL: test_v4i8_u32
-; CHECK:            ld.param.u32    [[R:%r[0-9]+]], [test_v4i8_u32_param_0];
-; CHECK-DAG:        bfe.u32         [[R0:%r[0-9]+]], [[R]], 0, 8;
-; CHECK-DAG:        bfe.u32         [[R1:%r[0-9]+]], [[R]], 8, 8;
-; CHECK-DAG:        bfe.u32         [[R2:%r[0-9]+]], [[R]], 16, 8;
-; CHECK-DAG:        bfe.u32         [[R3:%r[0-9]+]], [[R]], 24, 8;
-; CHECK-DAG:        add.s32         [[R01:%r[0-9]+]], [[R0]], [[R1]]
-; CHECK-DAG:        add.s32         [[R23:%r[0-9]+]], [[R2]], [[R3]]
-; CHECK-DAG:        add.s32         [[R0123:%r[0-9]+]], [[R01]], [[R23]]
 define i32  @test_v4i8_u32(i32 %a) {
+; CHECK-LABEL: test_v4i8_u32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u32 %r1, [test_v4i8_u32_param_0];
+; CHECK-NEXT:    bfe.u32 %r2, %r1, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r3, %r1, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r4, %r1, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r5, %r1, 24, 8;
+; CHECK-NEXT:    add.s32 %r6, %r2, %r3;
+; CHECK-NEXT:    add.s32 %r7, %r4, %r5;
+; CHECK-NEXT:    add.s32 %r8, %r6, %r7;
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r8;
+; CHECK-NEXT:    ret;
   %v = bitcast i32 %a to <4 x i8>
   %r0 = extractelement <4 x i8> %v, i64 0
   %r1 = extractelement <4 x i8> %v, i64 1
@@ -94,28 +153,43 @@ define i32  @test_v4i8_u32(i32 %a) {
 
 
 
-; CHECK-LABEL: test_v8i8
-; CHECK:       ld.param.u64    [[R:%rd[0-9]+]], [test_v8i8_param_0];
-; CHECK-DAG:        cvt.u32.u64     [[R00:%r[0-9]+]], [[R]];
-; CHECK-DAG:        { .reg .b32 tmp; mov.b64 {tmp, [[R01:%r[0-9]+]]}, [[R]]; }
-; CHECK-DAG:        bfe.s32         [[R1:%r[0-9]+]], [[R00]], 0, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E1:%rs[0-9]+]], [[R1]];
-; CHECK-DAG:        bfe.s32         [[R2:%r[0-9]+]], [[R00]], 8, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E2:%rs[0-9]+]], [[R2]];
-; CHECK-DAG:        bfe.s32         [[R3:%r[0-9]+]], [[R00]], 16, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E3:%rs[0-9]+]], [[R3]];
-; CHECK-DAG:        bfe.s32         [[R4:%r[0-9]+]], [[R00]], 24, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E4:%rs[0-9]+]], [[R4]];
-; CHECK-DAG:        bfe.s32         [[R5:%r[0-9]+]], [[R01]], 0, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E5:%rs[0-9]+]], [[R5]];
-; CHECK-DAG:        bfe.s32         [[R6:%r[0-9]+]], [[R01]], 8, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E6:%rs[0-9]+]], [[R6]];
-; CHECK-DAG:        bfe.s32         [[R7:%r[0-9]+]], [[R01]], 16, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E7:%rs[0-9]+]], [[R7]];
-; CHECK-DAG:        bfe.s32         [[R8:%r[0-9]+]], [[R01]], 24, 8;
-; CHECK-DAG:        cvt.s8.s32      [[E8:%rs[0-9]+]], [[R8]];
-
 define i16  @test_v8i8(i64 %a) {
+; CHECK-LABEL: test_v8i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<16>;
+; CHECK-NEXT:    .reg .b32 %r<14>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_v8i8_param_0];
+; CHECK-NEXT:    cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r2}, %rd1; }
+; CHECK-NEXT:    bfe.s32 %r5, %r1, 0, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs1, %r5;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 8, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs2, %r6;
+; CHECK-NEXT:    bfe.s32 %r7, %r1, 16, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs3, %r7;
+; CHECK-NEXT:    bfe.s32 %r8, %r1, 24, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs4, %r8;
+; CHECK-NEXT:    bfe.s32 %r9, %r2, 0, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs5, %r9;
+; CHECK-NEXT:    bfe.s32 %r10, %r2, 8, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs6, %r10;
+; CHECK-NEXT:    bfe.s32 %r11, %r2, 16, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs7, %r11;
+; CHECK-NEXT:    bfe.s32 %r12, %r2, 24, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs8, %r12;
+; CHECK-NEXT:    add.s16 %rs9, %rs1, %rs2;
+; CHECK-NEXT:    add.s16 %rs10, %rs3, %rs4;
+; CHECK-NEXT:    add.s16 %rs11, %rs5, %rs6;
+; CHECK-NEXT:    add.s16 %rs12, %rs7, %rs8;
+; CHECK-NEXT:    add.s16 %rs13, %rs9, %rs10;
+; CHECK-NEXT:    add.s16 %rs14, %rs11, %rs12;
+; CHECK-NEXT:    add.s16 %rs15, %rs13, %rs14;
+; CHECK-NEXT:    cvt.u32.u16 %r13, %rs15;
+; CHECK-NEXT:    st.param.b32 [func_retval0+0], %r13;
+; CHECK-NEXT:    ret;
   %v = bitcast i64 %a to <8 x i8>
   %r0 = extractelement <8 x i8> %v, i64 0
   %r1 = extractelement <8 x i8> %v, i64 1
diff --git a/llvm/test/CodeGen/PowerPC/aix-vsx-splatimm.ll b/llvm/test/CodeGen/PowerPC/aix-vsx-splatimm.ll
index 05edf92d724988e..f5515e8ba19bd5b 100644
--- a/llvm/test/CodeGen/PowerPC/aix-vsx-splatimm.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-vsx-splatimm.ll
@@ -14,7 +14,9 @@ define void @test_aix_splatimm(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-AIX-NEXT:    slwi 3, 3, 8
 ; CHECK-AIX-NEXT:    neg 3, 3
 ; CHECK-AIX-NEXT:    lwz 6, 0(3)
-; CHECK-AIX-NEXT:    sth 3, -16(1)
+; CHECK-AIX-NEXT:    sldi 3, 3, 48
+; CHECK-AIX-NEXT:    std 3, -16(1)
+; CHECK-AIX-NEXT:    std 3, -8(1)
 ; CHECK-AIX-NEXT:    addi 3, 1, -16
 ; CHECK-AIX-NEXT:    lxvw4x 34, 0, 3
 ; CHECK-AIX-NEXT:    srwi 3, 4, 16
@@ -24,9 +26,11 @@ define void @test_aix_splatimm(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-AIX-NEXT:    mullw 3, 3, 4
 ; CHECK-AIX-NEXT:    li 4, 0
 ; CHECK-AIX-NEXT:    neg 3, 3
+; CHECK-AIX-NEXT:    sldi 3, 3, 48
 ; CHECK-AIX-NEXT:    vsplth 2, 2, 0
 ; CHECK-AIX-NEXT:    stxvw4x 34, 0, 4
-; CHECK-AIX-NEXT:    sth 3, -32(1)
+; CHECK-AIX-NEXT:    std 3, -32(1)
+; CHECK-AIX-NEXT:    std 3, -24(1)
 ; CHECK-AIX-NEXT:    addi 3, 1, -32
 ; CHECK-AIX-NEXT:    lxvw4x 34, 0, 3
 ; CHECK-AIX-NEXT:    vsplth 2, 2, 0
diff --git a/llvm/test/CodeGen/PowerPC/build-vector-from-load-and-zeros.ll b/llvm/test/CodeGen/PowerPC/build-vector-from-load-and-zeros.ll
index cc32a76b22c2872..6d35a7281de6b49 100644
--- a/llvm/test/CodeGen/PowerPC/build-vector-from-load-and-zeros.ll
+++ b/llvm/test/CodeGen/PowerPC/build-vector-from-load-and-zeros.ll
@@ -338,17 +338,16 @@ define <4 x i32> @build_v4i32_load_0(ptr nocapture noundef readonly %p) {
 ; PWR7-BE-LABEL: build_v4i32_load_0:
 ; PWR7-BE:       # %bb.0: # %entry
 ; PWR7-BE-NEXT:    lwz 3, 0(3)
-; PWR7-BE-NEXT:    li 4, 0
-; PWR7-BE-NEXT:    stw 4, -16(1)
-; PWR7-BE-NEXT:    stw 3, -32(1)
+; PWR7-BE-NEXT:    xxlxor 36, 36, 36
+; PWR7-BE-NEXT:    sldi 3, 3, 32
+; PWR7-BE-NEXT:    std 3, -32(1)
+; PWR7-BE-NEXT:    std 3, -24(1)
 ; PWR7-BE-NEXT:    addis 3, 2, .LCPI8_0@toc@ha
 ; PWR7-BE-NEXT:    addi 3, 3, .LCPI8_0@toc@l
 ; PWR7-BE-NEXT:    lxvw4x 34, 0, 3
-; PWR7-BE-NEXT:    addi 3, 1, -16
-; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
 ; PWR7-BE-NEXT:    addi 3, 1, -32
-; PWR7-BE-NEXT:    lxvw4x 36, 0, 3
-; PWR7-BE-NEXT:    vperm 2, 4, 3, 2
+; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
+; PWR7-BE-NEXT:    vperm 2, 3, 4, 2
 ; PWR7-BE-NEXT:    blr
 ;
 ; PWR8-BE-LABEL: build_v4i32_load_0:
@@ -402,17 +401,16 @@ define <4 x i32> @build_v4i32_load_1(ptr nocapture noundef readonly %p) {
 ; PWR7-BE-LABEL: build_v4i32_load_1:
 ; PWR7-BE:       # %bb.0: # %entry
 ; PWR7-BE-NEXT:    lwz 3, 0(3)
-; PWR7-BE-NEXT:    li 4, 0
-; PWR7-BE-NEXT:    stw 4, -32(1)
-; PWR7-BE-NEXT:    stw 3, -16(1)
+; PWR7-BE-NEXT:    xxlxor 36, 36, 36
+; PWR7-BE-NEXT:    sldi 3, 3, 32
+; PWR7-BE-NEXT:    std 3, -16(1)
+; PWR7-BE-NEXT:    std 3, -8(1)
 ; PWR7-BE-NEXT:    addis 3, 2, .LCPI9_0@toc@ha
 ; PWR7-BE-NEXT:    addi 3, 3, .LCPI9_0@toc@l
 ; PWR7-BE-NEXT:    lxvw4x 34, 0, 3
-; PWR7-BE-NEXT:    addi 3, 1, -32
-; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
 ; PWR7-BE-NEXT:    addi 3, 1, -16
-; PWR7-BE-NEXT:    lxvw4x 36, 0, 3
-; PWR7-BE-NEXT:    vperm 2, 3, 4, 2
+; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
+; PWR7-BE-NEXT:    vperm 2, 4, 3, 2
 ; PWR7-BE-NEXT:    blr
 ;
 ; PWR8-BE-LABEL: build_v4i32_load_1:
@@ -466,17 +464,16 @@ define <4 x i32> @build_v4i32_load_2(ptr nocapture noundef readonly %p) {
 ; PWR7-BE-LABEL: build_v4i32_load_2:
 ; PWR7-BE:       # %bb.0: # %entry
 ; PWR7-BE-NEXT:    lwz 3, 0(3)
-; PWR7-BE-NEXT:    li 4, 0
-; PWR7-BE-NEXT:    stw 4, -32(1)
-; PWR7-BE-NEXT:    stw 3, -16(1)
+; PWR7-BE-NEXT:    xxlxor 36, 36, 36
+; PWR7-BE-NEXT:    sldi 3, 3, 32
+; PWR7-BE-NEXT:    std 3, -16(1)
+; PWR7-BE-NEXT:    std 3, -8(1)
 ; PWR7-BE-NEXT:    addis 3, 2, .LCPI10_0@toc@ha
 ; PWR7-BE-NEXT:    addi 3, 3, .LCPI10_0@toc@l
 ; PWR7-BE-NEXT:    lxvw4x 34, 0, 3
-; PWR7-BE-NEXT:    addi 3, 1, -32
-; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
 ; PWR7-BE-NEXT:    addi 3, 1, -16
-; PWR7-BE-NEXT:    lxvw4x 36, 0, 3
-; PWR7-BE-NEXT:    vperm 2, 3, 4, 2
+; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
+; PWR7-BE-NEXT:    vperm 2, 4, 3, 2
 ; PWR7-BE-NEXT:    blr
 ;
 ; PWR8-BE-LABEL: build_v4i32_load_2:
@@ -530,17 +527,16 @@ define <4 x i32> @build_v4i32_load_3(ptr nocapture noundef readonly %p) {
 ; PWR7-BE-LABEL: build_v4i32_load_3:
 ; PWR7-BE:       # %bb.0: # %entry
 ; PWR7-BE-NEXT:    lwz 3, 0(3)
-; PWR7-BE-NEXT:    li 4, 0
-; PWR7-BE-NEXT:    stw 4, -32(1)
-; PWR7-BE-NEXT:    stw 3, -16(1)
+; PWR7-BE-NEXT:    xxlxor 36, 36, 36
+; PWR7-BE-NEXT:    sldi 3, 3, 32
+; PWR7-BE-NEXT:    std 3, -16(1)
+; PWR7-BE-NEXT:    std 3, -8(1)
 ; PWR7-BE-NEXT:    addis 3, 2, .LCPI11_0@toc@ha
 ; PWR7-BE-NEXT:    addi 3, 3, .LCPI11_0@toc@l
 ; PWR7-BE-NEXT:    lxvw4x 34, 0, 3
-; PWR7-BE-NEXT:    addi 3, 1, -32
-; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
 ; PWR7-BE-NEXT:    addi 3, 1, -16
-; PWR7-BE-NEXT:    lxvw4x 36, 0, 3
-; PWR7-BE-NEXT:    vperm 2, 3, 4, 2
+; PWR7-BE-NEXT:    lxvw4x 35, 0, 3
+; PWR7-BE-NEXT:    vperm 2, 4, 3, 2
 ; PWR7-BE-NEXT:    blr
 ;
 ; PWR8-BE-LABEL: build_v4i32_load_3:
diff --git a/llvm/test/CodeGen/PowerPC/load-and-splat.ll b/llvm/test/CodeGen/PowerPC/load-and-splat.ll
index 6d2f3b3abc42dc6..bc68ad2a67bf5d3 100644
--- a/llvm/test/CodeGen/PowerPC/load-and-splat.ll
+++ b/llvm/test/CodeGen/PowerPC/load-and-splat.ll
@@ -591,6 +591,7 @@ define <16 x i8> @adjusted_lxvwsx(ptr %s, ptr %t) {
 ; P7:       # %bb.0: # %entry
 ; P7-NEXT:    ld r3, 0(r3)
 ; P7-NEXT:    std r3, -16(r1)
+; P7-NEXT:    std r3, -8(r1)
 ; P7-NEXT:    addi r3, r1, -16
 ; P7-NEXT:    lxvw4x vs0, 0, r3
 ; P7-NEXT:    xxspltw v2, vs0, 1
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll b/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
index 41bd93e84697e74..64a2e89e34811e4 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
@@ -764,8 +764,18 @@ define <16 x i8> @sub_absv_8_ext(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr
 ;
 ; CHECK-PWR7-LABEL: sub_absv_8_ext:
 ; CHECK-PWR7:       # %bb.0: # %entry
-; CHECK-PWR7-NEXT:    stdu r1, -400(r1)
-; CHECK-PWR7-NEXT:    .cfi_def_cfa_offset 400
+; CHECK-PWR7-NEXT:    stdu r1, -512(r1)
+; CHECK-PWR7-NEXT:    .cfi_def_cfa_offset 512
+; CHECK-PWR7-NEXT:    .cfi_offset r14, -144
+; CHECK-PWR7-NEXT:    .cfi_offset r15, -136
+; CHECK-PWR7-NEXT:    .cfi_offset r16, -128
+; CHECK-PWR7-NEXT:    .cfi_offset r17, -120
+; CHECK-PWR7-NEXT:    .cfi_offset r18, -112
+; CHECK-PWR7-NEXT:    .cfi_offset r19, -104
+; CHECK-PWR7-NEXT:    .cfi_offset r20, -96
+; CHECK-PWR7-NEXT:    .cfi_offset r21, -88
+; CHECK-PWR7-NEXT:    .cfi_offset r22, -80
+; CHECK-PWR7-NEXT:    .cfi_offset r23, -72
 ; CHECK-PWR7-NEXT:    .cfi_offset r24, -64
 ; CHECK-PWR7-NEXT:    .cfi_offset r25, -56
 ; CHECK-PWR7-NEXT:    .cfi_offset r26, -48
@@ -773,184 +783,244 @@ define <16 x i8> @sub_absv_8_ext(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr
 ; CHECK-PWR7-NEXT:    .cfi_offset r28, -32
 ; CHECK-PWR7-NEXT:    .cfi_offset r29, -24
 ; CHECK-PWR7-NEXT:    .cfi_offset r30, -16
-; CHECK-PWR7-NEXT:    addi r3, r1, 304
-; CHECK-PWR7-NEXT:    std r24, 336(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r25, 344(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r26, 352(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r27, 360(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r28, 368(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r29, 376(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r30, 384(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    stxvw4x v2, 0, r3
+; CHECK-PWR7-NEXT:    .cfi_offset r31, -8
+; CHECK-PWR7-NEXT:    .cfi_offset r2, -152
 ; CHECK-PWR7-NEXT:    addi r3, r1, 320
-; CHECK-PWR7-NEXT:    lbz r4, 304(r1)
-; CHECK-PWR7-NEXT:    stxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    lbz r5, 305(r1)
-; CHECK-PWR7-NEXT:    lbz r6, 321(r1)
-; CHECK-PWR7-NEXT:    lbz r7, 306(r1)
-; CHECK-PWR7-NEXT:    lbz r8, 322(r1)
-; CHECK-PWR7-NEXT:    lbz r9, 307(r1)
-; CHECK-PWR7-NEXT:    lbz r10, 323(r1)
-; CHECK-PWR7-NEXT:    lbz r0, 309(r1)
-; CHECK-PWR7-NEXT:    lbz r30, 325(r1)
-; CHECK-PWR7-NEXT:    lbz r29, 310(r1)
-; CHECK-PWR7-NEXT:    lbz r28, 326(r1)
-; CHECK-PWR7-NEXT:    lbz r11, 308(r1)
-; CHECK-PWR7-NEXT:    lbz r12, 324(r1)
-; CHECK-PWR7-NEXT:    lbz r27, 311(r1)
-; CHECK-PWR7-NEXT:    lbz r26, 327(r1)
-; CHECK-PWR7-NEXT:    lbz r25, 312(r1)
-; CHECK-PWR7-NEXT:    sub r5, r5, r6
-; CHECK-PWR7-NEXT:    sub r6, r7, r8
-; CHECK-PWR7-NEXT:    sub r7, r9, r10
-; CHECK-PWR7-NEXT:    sub r9, r0, r30
-; CHECK-PWR7-NEXT:    sub r10, r29, r28
-; CHECK-PWR7-NEXT:    sub r8, r11, r12
-; CHECK-PWR7-NEXT:    srawi r0, r5, 31
-; CHECK-PWR7-NEXT:    srawi r30, r6, 31
-; CHECK-PWR7-NEXT:    srawi r29, r7, 31
-; CHECK-PWR7-NEXT:    srawi r28, r8, 31
-; CHECK-PWR7-NEXT:    sub r11, r27, r26
-; CHECK-PWR7-NEXT:    srawi r27, r9, 31
-; CHECK-PWR7-NEXT:    lbz r24, 328(r1)
-; CHECK-PWR7-NEXT:    xor r5, r5, r0
-; CHECK-PWR7-NEXT:    xor r6, r6, r30
-; CHECK-PWR7-NEXT:    xor r7, r7, r29
-; CHECK-PWR7-NEXT:    xor r8, r8, r28
-; CHECK-PWR7-NEXT:    xor r9, r9, r27
-; CHECK-PWR7-NEXT:    srawi r26, r10, 31
-; CHECK-PWR7-NEXT:    sub r5, r5, r0
-; CHECK-PWR7-NEXT:    sub r6, r6, r30
-; CHECK-PWR7-NEXT:    lbz r0, 313(r1)
-; CHECK-PWR7-NEXT:    lbz r30, 329(r1)
-; CHECK-PWR7-NEXT:    sub r7, r7, r29
-; CHECK-PWR7-NEXT:    lbz r29, 330(r1)
-; CHECK-PWR7-NEXT:    sub r8, r8, r28
-; CHECK-PWR7-NEXT:    lbz r28, 331(r1)
-; CHECK-PWR7-NEXT:    sub r9, r9, r27
-; CHECK-PWR7-NEXT:    lbz r27, 332(r1)
-; CHECK-PWR7-NEXT:    xor r10, r10, r26
-; CHECK-PWR7-NEXT:    sub r10, r10, r26
-; CHECK-PWR7-NEXT:    lbz r26, 333(r1)
-; CHECK-PWR7-NEXT:    sub r12, r25, r24
-; CHECK-PWR7-NEXT:    srawi r25, r11, 31
+; CHECK-PWR7-NEXT:    std r14, 368(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r15, 376(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r16, 384(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r17, 392(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r18, 400(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r19, 408(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r20, 416(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r21, 424(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r22, 432(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r23, 440(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r24, 448(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r25, 456(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r26, 464(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r27, 472(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r28, 480(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r29, 488(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r30, 496(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r31, 504(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r2, 360(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    stxvw4x v2, 0, r3
 ; CHECK-PWR7-NEXT:    lbz r3, 320(r1)
+; CHECK-PWR7-NEXT:    addi r4, r1, 336
+; CHECK-PWR7-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; CHECK-PWR7-NEXT:    stxvw4x v3, 0, r4
+; CHECK-PWR7-NEXT:    lbz r15, 334(r1)
+; CHECK-PWR7-NEXT:    lbz r14, 350(r1)
+; CHECK-PWR7-NEXT:    lbz r31, 335(r1)
+; CHECK-PWR7-NEXT:    lbz r2, 351(r1)
+; CHECK-PWR7-NEXT:    sub r15, r15, r14
+; CHECK-PWR7-NEXT:    sub r14, r31, r2
+; CHECK-PWR7-NEXT:    srawi r2, r14, 31
+; CHECK-PWR7-NEXT:    xor r14, r14, r2
+; CHECK-PWR7-NEXT:    lbz r3, 333(r1)
+; CHECK-PWR7-NEXT:    lbz r19, 331(r1)
+; CHECK-PWR7-NEXT:    lbz r18, 347(r1)
+; CHECK-PWR7-NEXT:    sub r19, r19, r18
+; CHECK-PWR7-NEXT:    lbz r17, 332(r1)
+; CHECK-PWR7-NEXT:    lbz r16, 348(r1)
+; CHECK-PWR7-NEXT:    sub r17, r17, r16
+; CHECK-PWR7-NEXT:    lbz r23, 329(r1)
+; CHECK-PWR7-NEXT:    sub r14, r14, r2
+; CHECK-PWR7-NEXT:    lbz r2, 349(r1)
+; CHECK-PWR7-NEXT:    lbz r22, 345(r1)
+; CHECK-PWR7-NEXT:    lbz r4, 336(r1)
+; CHECK-PWR7-NEXT:    lbz r5, 321(r1)
+; CHECK-PWR7-NEXT:    lbz r6, 337(r1)
+; CHECK-PWR7-NEXT:    lbz r7, 322(r1)
+; CHECK-PWR7-NEXT:    lbz r8, 338(r1)
+; CHECK-PWR7-NEXT:    lbz r9, 323(r1)
+; CHECK-PWR7-NEXT:    lbz r10, 339(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 324(r1)
+; CHECK-PWR7-NEXT:    lbz r12, 340(r1)
+; CHECK-PWR7-NEXT:    lbz r0, 325(r1)
+; CHECK-PWR7-NEXT:    lbz r30, 341(r1)
+; CHECK-PWR7-NEXT:    lbz r29, 326(r1)
+; CHECK-PWR7-NEXT:    lbz r28, 342(r1)
+; CHECK-PWR7-NEXT:    lbz r27, 327(r1)
+; CHECK-PWR7-NEXT:    lbz r26, 343(r1)
+; CHECK-PWR7-NEXT:    sub r3, r3, r2
+; CHECK-PWR7-NEXT:    lbz r25, 328(r1)
+; CHECK-PWR7-NEXT:    lbz r24, 344(r1)
+; CHECK-PWR7-NEXT:    lbz r21, 330(r1)
+; CHECK-PWR7-NEXT:    lbz r20, 346(r1)
+; CHECK-PWR7-NEXT:    sub r5, r5, r6
+; CHECK-PWR7-NEXT:    srawi r18, r3, 31
+; CHECK-PWR7-NEXT:    sub r7, r7, r8
+; CHECK-PWR7-NEXT:    sub r9, r9, r10
+; CHECK-PWR7-NEXT:    sub r11, r11, r12
 ; CHECK-PWR7-NEXT:    sub r0, r0, r30
-; CHECK-PWR7-NEXT:    xor r11, r11, r25
-; CHECK-PWR7-NEXT:    sub r11, r11, r25
-; CHECK-PWR7-NEXT:    lbz r25, 334(r1)
-; CHECK-PWR7-NEXT:    sub r4, r4, r3
-; CHECK-PWR7-NEXT:    srawi r30, r0, 31
-; CHECK-PWR7-NEXT:    srawi r24, r12, 31
-; CHECK-PWR7-NEXT:    xor r12, r12, r24
-; CHECK-PWR7-NEXT:    sub r12, r12, r24
-; CHECK-PWR7-NEXT:    lbz r24, 335(r1)
-; CHECK-PWR7-NEXT:    srawi r3, r4, 31
-; CHECK-PWR7-NEXT:    xor r4, r4, r3
-; CHECK-PWR7-NEXT:    xor r0, r0, r30
-; CHECK-PWR7-NEXT:    sub r3, r4, r3
-; CHECK-PWR7-NEXT:    stb r3, 48(r1)
-; CHECK-PWR7-NEXT:    addi r3, r1, 288
-; CHECK-PWR7-NEXT:    stb r12, 176(r1)
-; CHECK-PWR7-NEXT:    sub r0, r0, r30
-; CHECK-PWR7-NEXT:    lbz r30, 314(r1)
-; CHECK-PWR7-NEXT:    stb r11, 160(r1)
-; CHECK-PWR7-NEXT:    sub r30, r30, r29
-; CHECK-PWR7-NEXT:    stb r0, 192(r1)
-; CHECK-PWR7-NEXT:    stb r10, 144(r1)
-; CHECK-PWR7-NEXT:    stb r9, 128(r1)
-; CHECK-PWR7-NEXT:    stb r8, 112(r1)
-; CHECK-PWR7-NEXT:    stb r7, 96(r1)
-; CHECK-PWR7-NEXT:    stb r6, 80(r1)
-; CHECK-PWR7-NEXT:    srawi r29, r30, 31
-; CHECK-PWR7-NEXT:    stb r5, 64(r1)
-; CHECK-PWR7-NEXT:    xor r30, r30, r29
-; CHECK-PWR7-NEXT:    sub r30, r30, r29
-; CHECK-PWR7-NEXT:    lbz r29, 315(r1)
 ; CHECK-PWR7-NEXT:    sub r29, r29, r28
-; CHECK-PWR7-NEXT:    stb r30, 208(r1)
-; CHECK-PWR7-NEXT:    ld r30, 384(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    srawi r28, r29, 31
-; CHECK-PWR7-NEXT:    xor r29, r29, r28
-; CHECK-PWR7-NEXT:    sub r29, r29, r28
-; CHECK-PWR7-NEXT:    lbz r28, 316(r1)
-; CHECK-PWR7-NEXT:    sub r28, r28, r27
-; CHECK-PWR7-NEXT:    stb r29, 224(r1)
-; CHECK-PWR7-NEXT:    ld r29, 376(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    srawi r27, r28, 31
-; CHECK-PWR7-NEXT:    xor r28, r28, r27
-; CHECK-PWR7-NEXT:    sub r28, r28, r27
-; CHECK-PWR7-NEXT:    lbz r27, 317(r1)
 ; CHECK-PWR7-NEXT:    sub r27, r27, r26
-; CHECK-PWR7-NEXT:    stb r28, 240(r1)
-; CHECK-PWR7-NEXT:    ld r28, 368(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    srawi r26, r27, 31
-; CHECK-PWR7-NEXT:    xor r27, r27, r26
-; CHECK-PWR7-NEXT:    sub r27, r27, r26
-; CHECK-PWR7-NEXT:    lbz r26, 318(r1)
-; CHECK-PWR7-NEXT:    sub r26, r26, r25
-; CHECK-PWR7-NEXT:    stb r27, 256(r1)
-; CHECK-PWR7-NEXT:    ld r27, 360(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    srawi r25, r26, 31
-; CHECK-PWR7-NEXT:    xor r26, r26, r25
-; CHECK-PWR7-NEXT:    sub r26, r26, r25
-; CHECK-PWR7-NEXT:    lbz r25, 319(r1)
 ; CHECK-PWR7-NEXT:    sub r25, r25, r24
-; CHECK-PWR7-NEXT:    stb r26, 272(r1)
-; CHECK-PWR7-NEXT:    ld r26, 352(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r31, r15, 31
+; CHECK-PWR7-NEXT:    ld r2, 360(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    xor r3, r3, r18
+; CHECK-PWR7-NEXT:    srawi r6, r5, 31
+; CHECK-PWR7-NEXT:    srawi r8, r7, 31
+; CHECK-PWR7-NEXT:    srawi r10, r9, 31
+; CHECK-PWR7-NEXT:    srawi r12, r11, 31
+; CHECK-PWR7-NEXT:    srawi r30, r0, 31
+; CHECK-PWR7-NEXT:    sub r3, r3, r18
+; CHECK-PWR7-NEXT:    srawi r18, r19, 31
+; CHECK-PWR7-NEXT:    srawi r28, r29, 31
+; CHECK-PWR7-NEXT:    ld r16, 384(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    srawi r26, r27, 31
 ; CHECK-PWR7-NEXT:    srawi r24, r25, 31
+; CHECK-PWR7-NEXT:    xor r19, r19, r18
+; CHECK-PWR7-NEXT:    xor r15, r15, r31
+; CHECK-PWR7-NEXT:    xor r5, r5, r6
+; CHECK-PWR7-NEXT:    std r3, 272(r1)
+; CHECK-PWR7-NEXT:    std r3, 280(r1)
+; CHECK-PWR7-NEXT:    srawi r3, r17, 31
+; CHECK-PWR7-NEXT:    sub r19, r19, r18
+; CHECK-PWR7-NEXT:    xor r7, r7, r8
+; CHECK-PWR7-NEXT:    sub r15, r15, r31
+; CHECK-PWR7-NEXT:    xor r17, r17, r3
+; CHECK-PWR7-NEXT:    xor r9, r9, r10
+; CHECK-PWR7-NEXT:    xor r11, r11, r12
+; CHECK-PWR7-NEXT:    xor r0, r0, r30
+; CHECK-PWR7-NEXT:    xor r29, r29, r28
+; CHECK-PWR7-NEXT:    xor r27, r27, r26
+; CHECK-PWR7-NEXT:    sub r3, r17, r3
 ; CHECK-PWR7-NEXT:    xor r25, r25, r24
 ; CHECK-PWR7-NEXT:    sub r25, r25, r24
-; CHECK-PWR7-NEXT:    ld r24, 336(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    stb r25, 288(r1)
-; CHECK-PWR7-NEXT:    ld r25, 344(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r27, r27, r26
+; CHECK-PWR7-NEXT:    sub r29, r29, r28
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    sub r0, r0, r30
+; CHECK-PWR7-NEXT:    sub r11, r11, r12
+; CHECK-PWR7-NEXT:    sub r9, r9, r10
+; CHECK-PWR7-NEXT:    sub r7, r7, r8
+; CHECK-PWR7-NEXT:    sub r5, r5, r6
+; CHECK-PWR7-NEXT:    sldi r14, r14, 56
+; CHECK-PWR7-NEXT:    sldi r15, r15, 56
+; CHECK-PWR7-NEXT:    ld r31, 504(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 256(r1)
+; CHECK-PWR7-NEXT:    std r3, 264(r1)
+; CHECK-PWR7-NEXT:    sldi r3, r19, 56
+; CHECK-PWR7-NEXT:    sldi r25, r25, 56
+; CHECK-PWR7-NEXT:    sldi r27, r27, 56
+; CHECK-PWR7-NEXT:    std r3, 240(r1)
+; CHECK-PWR7-NEXT:    std r3, 248(r1)
+; CHECK-PWR7-NEXT:    sub r3, r23, r22
+; CHECK-PWR7-NEXT:    srawi r23, r3, 31
+; CHECK-PWR7-NEXT:    sub r22, r21, r20
+; CHECK-PWR7-NEXT:    srawi r21, r22, 31
+; CHECK-PWR7-NEXT:    sldi r29, r29, 56
+; CHECK-PWR7-NEXT:    sldi r0, r0, 56
+; CHECK-PWR7-NEXT:    sldi r11, r11, 56
+; CHECK-PWR7-NEXT:    xor r3, r3, r23
+; CHECK-PWR7-NEXT:    xor r22, r22, r21
+; CHECK-PWR7-NEXT:    sldi r9, r9, 56
+; CHECK-PWR7-NEXT:    sldi r7, r7, 56
+; CHECK-PWR7-NEXT:    sldi r5, r5, 56
+; CHECK-PWR7-NEXT:    ld r30, 496(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    ld r28, 480(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r3, r3, r23
+; CHECK-PWR7-NEXT:    sub r22, r22, r21
+; CHECK-PWR7-NEXT:    std r14, 304(r1)
+; CHECK-PWR7-NEXT:    ld r26, 464(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    sldi r22, r22, 56
+; CHECK-PWR7-NEXT:    ld r24, 448(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    ld r23, 440(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r14, 312(r1)
+; CHECK-PWR7-NEXT:    std r15, 288(r1)
+; CHECK-PWR7-NEXT:    std r3, 208(r1)
+; CHECK-PWR7-NEXT:    std r3, 216(r1)
+; CHECK-PWR7-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r15, 296(r1)
+; CHECK-PWR7-NEXT:    ld r21, 424(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    ld r20, 416(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r22, 224(r1)
+; CHECK-PWR7-NEXT:    std r22, 232(r1)
+; CHECK-PWR7-NEXT:    sub r4, r3, r4
+; CHECK-PWR7-NEXT:    std r25, 192(r1)
+; CHECK-PWR7-NEXT:    ld r22, 432(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    ld r19, 408(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r3, r4, 31
+; CHECK-PWR7-NEXT:    std r25, 200(r1)
+; CHECK-PWR7-NEXT:    ld r25, 456(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r27, 176(r1)
+; CHECK-PWR7-NEXT:    std r27, 184(r1)
+; CHECK-PWR7-NEXT:    xor r4, r4, r3
+; CHECK-PWR7-NEXT:    std r29, 160(r1)
+; CHECK-PWR7-NEXT:    ld r27, 472(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r29, 168(r1)
+; CHECK-PWR7-NEXT:    std r0, 144(r1)
+; CHECK-PWR7-NEXT:    sub r3, r4, r3
+; CHECK-PWR7-NEXT:    std r0, 152(r1)
+; CHECK-PWR7-NEXT:    ld r29, 488(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    ld r18, 400(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    std r11, 128(r1)
+; CHECK-PWR7-NEXT:    ld r17, 392(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r11, 136(r1)
+; CHECK-PWR7-NEXT:    std r9, 112(r1)
+; CHECK-PWR7-NEXT:    std r3, 64(r1)
+; CHECK-PWR7-NEXT:    std r3, 72(r1)
+; CHECK-PWR7-NEXT:    addi r3, r1, 304
+; CHECK-PWR7-NEXT:    std r9, 120(r1)
+; CHECK-PWR7-NEXT:    ld r15, 376(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r7, 96(r1)
+; CHECK-PWR7-NEXT:    std r7, 104(r1)
+; CHECK-PWR7-NEXT:    std r5, 80(r1)
+; CHECK-PWR7-NEXT:    std r5, 88(r1)
 ; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 272
+; CHECK-PWR7-NEXT:    addi r3, r1, 288
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 256
+; CHECK-PWR7-NEXT:    addi r3, r1, 272
+; CHECK-PWR7-NEXT:    ld r14, 368(r1) # 8-byte Folded Reload
 ; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 240
+; CHECK-PWR7-NEXT:    addi r3, r1, 256
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 224
+; CHECK-PWR7-NEXT:    addi r3, r1, 240
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    vmrghh v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 208
+; CHECK-PWR7-NEXT:    addi r3, r1, 224
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 192
+; CHECK-PWR7-NEXT:    addi r3, r1, 208
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 176
+; CHECK-PWR7-NEXT:    addi r3, r1, 192
 ; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 160
+; CHECK-PWR7-NEXT:    addi r3, r1, 176
 ; CHECK-PWR7-NEXT:    vmrghb v4, v5, v4
 ; CHECK-PWR7-NEXT:    vmrghh v3, v4, v3
 ; CHECK-PWR7-NEXT:    xxmrghw vs0, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 144
+; CHECK-PWR7-NEXT:    addi r3, r1, 160
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 128
+; CHECK-PWR7-NEXT:    addi r3, r1, 144
 ; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 112
+; CHECK-PWR7-NEXT:    addi r3, r1, 128
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 96
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
+; CHECK-PWR7-NEXT:    addi r3, r1, 112
 ; CHECK-PWR7-NEXT:    vmrghh v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 80
+; CHECK-PWR7-NEXT:    addi r3, r1, 96
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 64
+; CHECK-PWR7-NEXT:    addi r3, r1, 80
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 48
+; CHECK-PWR7-NEXT:    addi r3, r1, 64
 ; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
 ; CHECK-PWR7-NEXT:    vmrghb v4, v5, v4
 ; CHECK-PWR7-NEXT:    vmrghh v3, v4, v3
 ; CHECK-PWR7-NEXT:    xxmrghw vs1, v3, v2
 ; CHECK-PWR7-NEXT:    xxmrghd v2, vs1, vs0
-; CHECK-PWR7-NEXT:    addi r1, r1, 400
+; CHECK-PWR7-NEXT:    addi r1, r1, 512
 ; CHECK-PWR7-NEXT:    blr
 entry:
   %vecext = extractelement <16 x i8> %a, i32 0
diff --git a/llvm/test/CodeGen/PowerPC/test-vector-insert.ll b/llvm/test/CodeGen/PowerPC/test-vector-insert.ll
index 087f2244f0f7d06..73b4ad8a507b82e 100644
--- a/llvm/test/CodeGen/PowerPC/test-vector-insert.ll
+++ b/llvm/test/CodeGen/PowerPC/test-vector-insert.ll
@@ -60,7 +60,9 @@ define dso_local <4 x i32> @test(<4 x i32> %a, double %b) {
 ; CHECK-BE-P7-NEXT:    addi r3, r1, -4
 ; CHECK-BE-P7-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-P7-NEXT:    lwz r3, -4(r1)
-; CHECK-BE-P7-NEXT:    stw r3, -32(r1)
+; CHECK-BE-P7-NEXT:    sldi r3, r3, 32
+; CHECK-BE-P7-NEXT:    std r3, -32(r1)
+; CHECK-BE-P7-NEXT:    std r3, -24(r1)
 ; CHECK-BE-P7-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
 ; CHECK-BE-P7-NEXT:    addi r3, r3, .LCPI0_0@toc@l
 ; CHECK-BE-P7-NEXT:    lxvw4x v3, 0, r3
@@ -129,7 +131,9 @@ define dso_local <4 x i32> @test2(<4 x i32> %a, float %b) {
 ; CHECK-BE-P7-NEXT:    addi r3, r1, -4
 ; CHECK-BE-P7-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-P7-NEXT:    lwz r3, -4(r1)
-; CHECK-BE-P7-NEXT:    stw r3, -32(r1)
+; CHECK-BE-P7-NEXT:    sldi r3, r3, 32
+; CHECK-BE-P7-NEXT:    std r3, -32(r1)
+; CHECK-BE-P7-NEXT:    std r3, -24(r1)
 ; CHECK-BE-P7-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
 ; CHECK-BE-P7-NEXT:    addi r3, r3, .LCPI1_0@toc@l
 ; CHECK-BE-P7-NEXT:    lxvw4x v3, 0, r3
@@ -198,7 +202,9 @@ define dso_local <4 x i32> @test3(<4 x i32> %a, double %b) {
 ; CHECK-BE-P7-NEXT:    addi r3, r1, -4
 ; CHECK-BE-P7-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-P7-NEXT:    lwz r3, -4(r1)
-; CHECK-BE-P7-NEXT:    stw r3, -32(r1)
+; CHECK-BE-P7-NEXT:    sldi r3, r3, 32
+; CHECK-BE-P7-NEXT:    std r3, -32(r1)
+; CHECK-BE-P7-NEXT:    std r3, -24(r1)
 ; CHECK-BE-P7-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
 ; CHECK-BE-P7-NEXT:    addi r3, r3, .LCPI2_0@toc@l
 ; CHECK-BE-P7-NEXT:    lxvw4x v3, 0, r3
@@ -267,7 +273,9 @@ define dso_local <4 x i32> @test4(<4 x i32> %a, float %b) {
 ; CHECK-BE-P7-NEXT:    addi r3, r1, -4
 ; CHECK-BE-P7-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-P7-NEXT:    lwz r3, -4(r1)
-; CHECK-BE-P7-NEXT:    stw r3, -32(r1)
+; CHECK-BE-P7-NEXT:    sldi r3, r3, 32
+; CHECK-BE-P7-NEXT:    std r3, -32(r1)
+; CHECK-BE-P7-NEXT:    std r3, -24(r1)
 ; CHECK-BE-P7-NEXT:    addis r3, r2, .LCPI3_0@toc@ha
 ; CHECK-BE-P7-NEXT:    addi r3, r3, .LCPI3_0@toc@l
 ; CHECK-BE-P7-NEXT:    lxvw4x v3, 0, r3
diff --git a/llvm/test/CodeGen/PowerPC/vec-trunc2.ll b/llvm/test/CodeGen/PowerPC/vec-trunc2.ll
index 7a09d5a5e8bb2e2..b81bc9f21423f3c 100644
--- a/llvm/test/CodeGen/PowerPC/vec-trunc2.ll
+++ b/llvm/test/CodeGen/PowerPC/vec-trunc2.ll
@@ -28,16 +28,31 @@ define dso_local <8 x i8> @test8x32(i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5,
 ;
 ; CHECK-BE-LABEL: test8x32:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    stw r10, -80(r1)
-; CHECK-BE-NEXT:    stw r9, -96(r1)
-; CHECK-BE-NEXT:    stw r8, -112(r1)
-; CHECK-BE-NEXT:    stw r7, -128(r1)
-; CHECK-BE-NEXT:    stw r6, -16(r1)
-; CHECK-BE-NEXT:    stw r5, -32(r1)
-; CHECK-BE-NEXT:    stw r4, -48(r1)
-; CHECK-BE-NEXT:    stw r3, -64(r1)
-; CHECK-BE-NEXT:    addi r3, r1, -80
-; CHECK-BE-NEXT:    lxvw4x vs0, 0, r3
+; CHECK-BE-NEXT:    sldi r10, r10, 32
+; CHECK-BE-NEXT:    sldi r9, r9, 32
+; CHECK-BE-NEXT:    sldi r8, r8, 32
+; CHECK-BE-NEXT:    sldi r7, r7, 32
+; CHECK-BE-NEXT:    sldi r6, r6, 32
+; CHECK-BE-NEXT:    sldi r5, r5, 32
+; CHECK-BE-NEXT:    sldi r4, r4, 32
+; CHECK-BE-NEXT:    sldi r3, r3, 32
+; CHECK-BE-NEXT:    addi r11, r1, -80
+; CHECK-BE-NEXT:    std r10, -80(r1)
+; CHECK-BE-NEXT:    std r10, -72(r1)
+; CHECK-BE-NEXT:    std r9, -96(r1)
+; CHECK-BE-NEXT:    std r9, -88(r1)
+; CHECK-BE-NEXT:    std r8, -112(r1)
+; CHECK-BE-NEXT:    std r8, -104(r1)
+; CHECK-BE-NEXT:    std r7, -128(r1)
+; CHECK-BE-NEXT:    std r7, -120(r1)
+; CHECK-BE-NEXT:    std r6, -16(r1)
+; CHECK-BE-NEXT:    std r6, -8(r1)
+; CHECK-BE-NEXT:    std r5, -32(r1)
+; CHECK-BE-NEXT:    std r5, -24(r1)
+; CHECK-BE-NEXT:    std r4, -48(r1)
+; CHECK-BE-NEXT:    std r4, -40(r1)
+; CHECK-BE-NEXT:    std r3, -64(r1)
+; CHECK-BE-NEXT:    std r3, -56(r1)
 ; CHECK-BE-NEXT:    addi r3, r1, -96
 ; CHECK-BE-NEXT:    lxvw4x vs1, 0, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -112
@@ -51,15 +66,16 @@ define dso_local <8 x i8> @test8x32(i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5,
 ; CHECK-BE-NEXT:    addi r3, r1, -48
 ; CHECK-BE-NEXT:    lxvw4x vs6, 0, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -64
+; CHECK-BE-NEXT:    lxvw4x vs0, 0, r11
 ; CHECK-BE-NEXT:    lxvw4x vs7, 0, r3
 ; CHECK-BE-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI0_0@toc@l
 ; CHECK-BE-NEXT:    xxmrghw vs0, vs1, vs0
 ; CHECK-BE-NEXT:    xxmrghw vs1, vs3, vs2
 ; CHECK-BE-NEXT:    xxmrghw vs2, vs5, vs4
-; CHECK-BE-NEXT:    addi r3, r3, .LCPI0_0@toc@l
-; CHECK-BE-NEXT:    xxmrghd v3, vs1, vs0
 ; CHECK-BE-NEXT:    lxvw4x v2, 0, r3
 ; CHECK-BE-NEXT:    xxmrghw vs3, vs7, vs6
+; CHECK-BE-NEXT:    xxmrghd v3, vs1, vs0
 ; CHECK-BE-NEXT:    xxmrghd v4, vs3, vs2
 ; CHECK-BE-NEXT:    vperm v2, v4, v3, v2
 ; CHECK-BE-NEXT:    blr
@@ -136,16 +152,31 @@ define dso_local <8 x i16> @test8x24(i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5
 ;
 ; CHECK-BE-LABEL: test8x24:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    sth r10, -16(r1)
-; CHECK-BE-NEXT:    sth r9, -32(r1)
-; CHECK-BE-NEXT:    sth r8, -48(r1)
-; CHECK-BE-NEXT:    sth r7, -64(r1)
-; CHECK-BE-NEXT:    sth r6, -80(r1)
-; CHECK-BE-NEXT:    sth r5, -96(r1)
-; CHECK-BE-NEXT:    sth r4, -112(r1)
-; CHECK-BE-NEXT:    sth r3, -128(r1)
-; CHECK-BE-NEXT:    addi r3, r1, -16
-; CHECK-BE-NEXT:    lxvw4x v2, 0, r3
+; CHECK-BE-NEXT:    sldi r10, r10, 48
+; CHECK-BE-NEXT:    sldi r9, r9, 48
+; CHECK-BE-NEXT:    sldi r8, r8, 48
+; CHECK-BE-NEXT:    sldi r7, r7, 48
+; CHECK-BE-NEXT:    sldi r6, r6, 48
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    addi r11, r1, -16
+; CHECK-BE-NEXT:    std r10, -16(r1)
+; CHECK-BE-NEXT:    std r10, -8(r1)
+; CHECK-BE-NEXT:    std r9, -32(r1)
+; CHECK-BE-NEXT:    std r9, -24(r1)
+; CHECK-BE-NEXT:    std r8, -48(r1)
+; CHECK-BE-NEXT:    std r8, -40(r1)
+; CHECK-BE-NEXT:    std r7, -64(r1)
+; CHECK-BE-NEXT:    std r7, -56(r1)
+; CHECK-BE-NEXT:    std r6, -80(r1)
+; CHECK-BE-NEXT:    std r6, -72(r1)
+; CHECK-BE-NEXT:    std r5, -96(r1)
+; CHECK-BE-NEXT:    std r5, -88(r1)
+; CHECK-BE-NEXT:    std r4, -112(r1)
+; CHECK-BE-NEXT:    std r4, -104(r1)
+; CHECK-BE-NEXT:    std r3, -128(r1)
+; CHECK-BE-NEXT:    std r3, -120(r1)
 ; CHECK-BE-NEXT:    addi r3, r1, -32
 ; CHECK-BE-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -48
@@ -159,6 +190,7 @@ define dso_local <8 x i16> @test8x24(i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5
 ; CHECK-BE-NEXT:    addi r3, r1, -112
 ; CHECK-BE-NEXT:    lxvw4x v6, 0, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -128
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r11
 ; CHECK-BE-NEXT:    lxvw4x v7, 0, r3
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
 ; CHECK-BE-NEXT:    vmrghh v3, v5, v4
diff --git a/llvm/test/CodeGen/PowerPC/vsx.ll b/llvm/test/CodeGen/PowerPC/vsx.ll
index d1d29a0f884c620..14b3d69f8c27306 100644
--- a/llvm/test/CodeGen/PowerPC/vsx.ll
+++ b/llvm/test/CodeGen/PowerPC/vsx.ll
@@ -2383,7 +2383,9 @@ define <2 x double> @test70(<2 x i8> %a) {
 define <2 x i32> @test80(i32 %v) {
 ; CHECK-LABEL: test80:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    stw r3, -16(r1)
+; CHECK-NEXT:    sldi r3, r3, 32
+; CHECK-NEXT:    std r3, -16(r1)
+; CHECK-NEXT:    std r3, -8(r1)
 ; CHECK-NEXT:    addi r3, r1, -16
 ; CHECK-NEXT:    lxvw4x vs0, 0, r3
 ; CHECK-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
@@ -2395,7 +2397,9 @@ define <2 x i32> @test80(i32 %v) {
 ;
 ; CHECK-REG-LABEL: test80:
 ; CHECK-REG:       # %bb.0:
-; CHECK-REG-NEXT:    stw r3, -16(r1)
+; CHECK-REG-NEXT:    sldi r3, r3, 32
+; CHECK-REG-NEXT:    std r3, -16(r1)
+; CHECK-REG-NEXT:    std r3, -8(r1)
 ; CHECK-REG-NEXT:    addi r3, r1, -16
 ; CHECK-REG-NEXT:    lxvw4x vs0, 0, r3
 ; CHECK-REG-NEXT:    addis r3, r2, .LCPI65_0@toc@ha
@@ -2407,8 +2411,12 @@ define <2 x i32> @test80(i32 %v) {
 ;
 ; CHECK-FISL-LABEL: test80:
 ; CHECK-FISL:       # %bb.0:
-; CHECK-FISL-NEXT:    # kill: def $r3 killed $r3 killed $x3
-; CHECK-FISL-NEXT:    stw r3, -16(r1)
+; CHECK-FISL-NEXT:    mr r4, r3
+; CHECK-FISL-NEXT:    # implicit-def: $x3
+; CHECK-FISL-NEXT:    mr r3, r4
+; CHECK-FISL-NEXT:    sldi r3, r3, 32
+; CHECK-FISL-NEXT:    std r3, -8(r1)
+; CHECK-FISL-NEXT:    std r3, -16(r1)
 ; CHECK-FISL-NEXT:    addi r3, r1, -16
 ; CHECK-FISL-NEXT:    lxvw4x vs0, 0, r3
 ; CHECK-FISL-NEXT:    xxspltw v2, vs0, 0
diff --git a/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll b/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
index c31b2faedd51dc5..069e734088a8454 100644
--- a/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
+++ b/llvm/test/CodeGen/PowerPC/widen-vec-correctly-be.ll
@@ -13,7 +13,9 @@ define void @test() local_unnamed_addr #0 align 2 {
 ; CHECK-BE-NEXT:    addi r3, r3, 1
 ; CHECK-BE-NEXT:    xxlxor vs1, vs1, vs1
 ; CHECK-BE-NEXT:    vsrw v2, v2, v2
-; CHECK-BE-NEXT:    sth r3, -32(r1)
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    std r3, -32(r1)
+; CHECK-BE-NEXT:    std r3, -24(r1)
 ; CHECK-BE-NEXT:    addi r3, r1, -32
 ; CHECK-BE-NEXT:    lxvw4x vs0, 0, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -16
diff --git a/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll b/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
index 219f0daf270bc14..23875a7ec562112 100644
--- a/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll
@@ -40,9 +40,9 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ;
 ; RV64IZbb-LABEL: func32:
 ; RV64IZbb:       # %bb.0:
+; RV64IZbb-NEXT:    sext.w a0, a0
 ; RV64IZbb-NEXT:    mulw a1, a1, a2
 ; RV64IZbb-NEXT:    not a2, a1
-; RV64IZbb-NEXT:    sext.w a0, a0
 ; RV64IZbb-NEXT:    minu a0, a0, a2
 ; RV64IZbb-NEXT:    add a0, a0, a1
 ; RV64IZbb-NEXT:    ret
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/length.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/length.ll
new file mode 100644
index 000000000000000..b4a9d8e0664b7e8
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/length.ll
@@ -0,0 +1,29 @@
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; Make sure SPIRV operation function calls for length are lowered correctly.
+
+; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#vec4_float_16:]] = OpTypeVector %[[#float_16]] 4
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+
+define noundef half @length_half4(<4 x half> noundef %a) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpExtInst %[[#float_16]] %[[#op_ext_glsl]] Length %[[#arg0]]
+  %hlsl.length = call half @llvm.spv.length.v4f16(<4 x half> %a)
+  ret half %hlsl.length
+}
+
+define noundef float @length_float4(<4 x float> noundef %a) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpExtInst %[[#float_32]] %[[#op_ext_glsl]] Length %[[#arg0]]
+  %hlsl.length = call float @llvm.spv.length.v4f32(<4 x float> %a)
+  ret float %hlsl.length
+}
+
+declare half @llvm.spv.length.v4f16(<4 x half>)
+declare float @llvm.spv.length.v4f32(<4 x float>)
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAllAny.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAllAny.ll
index 39f9e0581d2f8e2..0b420697763ba52 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAllAny.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/OpGroupAllAny.ll
@@ -8,11 +8,12 @@
 ; CHECK-SPIRV-DAG: %[[#BoolTypeID:]] = OpTypeBool
 ; CHECK-SPIRV-DAG: %[[#True:]] = OpConstantTrue %[[#BoolTypeID]]
 ; CHECK-SPIRV-DAG: %[[#False:]] = OpConstantFalse %[[#BoolTypeID]]
+
+; CHECK-SPIRV: OpFunction
 ; CHECK-SPIRV: %[[#]] = OpGroupAll %[[#BoolTypeID]] %[[#]] %[[#True]]
 ; CHECK-SPIRV: %[[#]] = OpGroupAny %[[#BoolTypeID]] %[[#]] %[[#True]]
 ; CHECK-SPIRV: %[[#]] = OpGroupAll %[[#BoolTypeID]] %[[#]] %[[#True]]
 ; CHECK-SPIRV: %[[#]] = OpGroupAny %[[#BoolTypeID]] %[[#]] %[[#False]]
-
 define spir_kernel void @test(i32 addrspace(1)* nocapture readnone %i) {
 entry:
   %call = tail call spir_func i32 @_Z14work_group_alli(i32 5)
@@ -22,8 +23,73 @@ entry:
   ret void
 }
 
-declare spir_func i32 @_Z14work_group_alli(i32)
-declare spir_func i32 @_Z14work_group_anyi(i32)
+; CHECK-SPIRV: OpFunction
+; CHECK-SPIRV: %[[#]] = OpGroupAny %[[#BoolTypeID]] %[[#]] %[[#]]
+; CHECK-SPIRV: %[[#]] = OpGroupAll %[[#BoolTypeID]] %[[#]] %[[#]]
+define spir_kernel void @test_nonconst_any(ptr addrspace(1) %input, ptr addrspace(1) %output)  #0 !kernel_arg_addr_space !7 !kernel_arg_access_qual !8 !kernel_arg_type !9 !kernel_arg_type_qual !10 !kernel_arg_base_type !9 !spirv.ParameterDecorations !11  {
+entry:
+  %r0 = call spir_func i64 @_Z13get_global_idj(i32 0)
+  %r1 = insertelement <3 x i64> undef, i64 %r0, i32 0
+  %r2 = call spir_func i64 @_Z13get_global_idj(i32 1)
+  %r3 = insertelement <3 x i64> %r1, i64 %r2, i32 1
+  %r4 = call spir_func i64 @_Z13get_global_idj(i32 2)
+  %r5 = insertelement <3 x i64> %r3, i64 %r4, i32 2
+  %call = extractelement <3 x i64> %r5, i32 0
+  %conv = trunc i64 %call to i32
+  %idxprom = sext i32 %conv to i64
+  %arrayidx = getelementptr inbounds float, ptr addrspace(1) %input, i64 %idxprom
+  %r6 = load float, ptr addrspace(1) %arrayidx, align 4
+  %add = add nsw i32 %conv, 1
+  %idxprom1 = sext i32 %add to i64
+  %arrayidx2 = getelementptr inbounds float, ptr addrspace(1) %input, i64 %idxprom1
+  %r7 = load float, ptr addrspace(1) %arrayidx2, align 4
+  %cmp = fcmp ogt float %r6, %r7
+  %conv3 = select i1 %cmp, i32 1, i32 0
+  %r8 = icmp ne i32 %conv3, 0
+  %r9 = zext i1 %r8 to i32
+  %r10 = call spir_func i32 @_Z14work_group_anyi(i32 %r9)
+  %call41 = icmp ne i32 %r10, 0
+  %call4 = select i1 %call41, i32 1, i32 0
+  %idxprom5 = sext i32 %conv to i64
+  %arrayidx6 = getelementptr inbounds i32, ptr addrspace(1) %output, i64 %idxprom5
+  store i32 %call4, ptr addrspace(1) %arrayidx6, align 4
+  %r11 = call spir_func i32 @_Z14work_group_alli(i32 %r9)
+  %call42 = icmp ne i32 %r11, 0
+  %call5 = select i1 %call42, i32 1, i32 0
+  store i32 %call5, ptr addrspace(1) %arrayidx6, align 4
+  ret void
+}
+
+declare spir_func i64 @_Z13get_global_idj(i32) #1
+
+declare spir_func i32 @_Z14work_group_alli(i32) #2
+declare spir_func i32 @_Z14work_group_anyi(i32) #2
 
 declare spir_func i1 @__spirv_GroupAll(i32, i1)
 declare spir_func i1 @__spirv_GroupAny(i32, i1)
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind willreturn memory(none) }
+attributes #2 = { convergent nounwind }
+
+!spirv.MemoryModel = !{!0}
+!opencl.enable.FP_CONTRACT = !{}
+!spirv.Source = !{!1}
+!opencl.spir.version = !{!2}
+!opencl.ocl.version = !{!3}
+!opencl.used.extensions = !{!4}
+!opencl.used.optional.core.features = !{!5}
+!spirv.Generator = !{!6}
+
+!0 = !{i32 2, i32 2}
+!1 = !{i32 3, i32 300000}
+!2 = !{i32 2, i32 0}
+!3 = !{i32 3, i32 0}
+!4 = !{!"cl_khr_subgroups"}
+!5 = !{}
+!6 = !{i16 6, i16 14}
+!7 = !{i32 1, i32 1}
+!8 = !{!"none", !"none"}
+!9 = !{!"float*", !"int*"}
+!10 = !{!"", !""}
+!11 = !{!5, !5}
diff --git a/llvm/test/CodeGen/X86/pr59305.ll b/llvm/test/CodeGen/X86/pr59305.ll
index 46c9da5a51939d0..d8738081842a336 100644
--- a/llvm/test/CodeGen/X86/pr59305.ll
+++ b/llvm/test/CodeGen/X86/pr59305.ll
@@ -81,20 +81,17 @@ define double @bar(double %0) #0 {
 ; X64-NEXT:    #APP
 ; X64-NEXT:    ldmxcsr 0
 ; X64-NEXT:    #NO_APP
-; X64-NEXT:    wait
 ; X64-NEXT:    movsd {{.*#+}} xmm2 = [1.0E+0,0.0E+0]
 ; X64-NEXT:    movapd %xmm2, %xmm3
 ; X64-NEXT:    divsd %xmm0, %xmm3
 ; X64-NEXT:    #APP
 ; X64-NEXT:    ldmxcsr 0
 ; X64-NEXT:    #NO_APP
-; X64-NEXT:    wait
 ; X64-NEXT:    movapd %xmm2, %xmm1
 ; X64-NEXT:    divsd %xmm0, %xmm1
 ; X64-NEXT:    #APP
 ; X64-NEXT:    ldmxcsr 0
 ; X64-NEXT:    #NO_APP
-; X64-NEXT:    wait
 ; X64-NEXT:    divsd %xmm0, %xmm2
 ; X64-NEXT:    movapd %xmm3, %xmm0
 ; X64-NEXT:    callq fma@PLT
@@ -105,6 +102,7 @@ define double @bar(double %0) #0 {
 ; X86:       # %bb.0:
 ; X86-NEXT:    subl $28, %esp
 ; X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-NEXT:    wait
 ; X86-NEXT:    #APP
 ; X86-NEXT:    fldcw 0
 ; X86-NEXT:    #NO_APP
diff --git a/llvm/test/MC/Disassembler/X86/apx/ccmp.txt b/llvm/test/MC/Disassembler/X86/apx/ccmp.txt
index d2656d62972d43b..766b3ecc25cf834 100644
--- a/llvm/test/MC/Disassembler/X86/apx/ccmp.txt
+++ b/llvm/test/MC/Disassembler/X86/apx/ccmp.txt
@@ -1181,6 +1181,110 @@
 # INTEL: ccmpne {dfv=of} r15, r9
 0x62,0x54,0xc4,0x05,0x3b,0xf9
 
+# ATT:   ccmpbeb {dfv=of} $123, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} byte ptr [r8 + 4*rax + 123], 123
+0x62,0xd4,0x44,0x06,0x80,0x7c,0x80,0x7b,0x7b
+
+# ATT:   ccmpbew {dfv=of} $123, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], 123
+0x62,0xd4,0x45,0x06,0x83,0x7c,0x80,0x7b,0x7b
+
+# ATT:   ccmpbew {dfv=of} $1234, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], 1234
+0x62,0xd4,0x45,0x06,0x81,0x7c,0x80,0x7b,0xd2,0x04
+
+# ATT:   ccmpbel {dfv=of} $123, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], 123
+0x62,0xd4,0x44,0x06,0x83,0x7c,0x80,0x7b,0x7b
+
+# ATT:   ccmpbel {dfv=of} $123456, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456
+0x62,0xd4,0x44,0x06,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00
+
+# ATT:   ccmpbeq {dfv=of} $123, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], 123
+0x62,0xd4,0xc4,0x06,0x83,0x7c,0x80,0x7b,0x7b
+
+# ATT:   ccmpbeq {dfv=of} $123456, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456
+0x62,0xd4,0xc4,0x06,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00
+
+# ATT:   ccmpbeb {dfv=of} %bl, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} byte ptr [r8 + 4*rax + 123], bl
+0x62,0xd4,0x44,0x06,0x38,0x5c,0x80,0x7b
+
+# ATT:   ccmpbew {dfv=of} %dx, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], dx
+0x62,0xd4,0x45,0x06,0x39,0x54,0x80,0x7b
+
+# ATT:   ccmpbel {dfv=of} %ecx, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx
+0x62,0xd4,0x44,0x06,0x39,0x4c,0x80,0x7b
+
+# ATT:   ccmpbeq {dfv=of} %r9, 123(%r8,%rax,4)
+# INTEL: ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], r9
+0x62,0x54,0xc4,0x06,0x39,0x4c,0x80,0x7b
+
+# ATT:   ccmpbeb {dfv=of} 123(%r8,%rax,4), %bl
+# INTEL: ccmpbe {dfv=of} bl, byte ptr [r8 + 4*rax + 123]
+0x62,0xd4,0x44,0x06,0x3a,0x5c,0x80,0x7b
+
+# ATT:   ccmpbew {dfv=of} 123(%r8,%rax,4), %dx
+# INTEL: ccmpbe {dfv=of} dx, word ptr [r8 + 4*rax + 123]
+0x62,0xd4,0x45,0x06,0x3b,0x54,0x80,0x7b
+
+# ATT:   ccmpbel {dfv=of} 123(%r8,%rax,4), %ecx
+# INTEL: ccmpbe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123]
+0x62,0xd4,0x44,0x06,0x3b,0x4c,0x80,0x7b
+
+# ATT:   ccmpbeq {dfv=of} 123(%r8,%rax,4), %r9
+# INTEL: ccmpbe {dfv=of} r9, qword ptr [r8 + 4*rax + 123]
+0x62,0x54,0xc4,0x06,0x3b,0x4c,0x80,0x7b
+
+# ATT:   ccmpbeb {dfv=of} $123, %bl
+# INTEL: ccmpbe {dfv=of} bl, 123
+0x62,0xf4,0x44,0x06,0x80,0xfb,0x7b
+
+# ATT:   ccmpbew {dfv=of} $123, %dx
+# INTEL: ccmpbe {dfv=of} dx, 123
+0x62,0xf4,0x45,0x06,0x83,0xfa,0x7b
+
+# ATT:   ccmpbel {dfv=of} $123, %ecx
+# INTEL: ccmpbe {dfv=of} ecx, 123
+0x62,0xf4,0x44,0x06,0x83,0xf9,0x7b
+
+# ATT:   ccmpbeq {dfv=of} $123, %r9
+# INTEL: ccmpbe {dfv=of} r9, 123
+0x62,0xd4,0xc4,0x06,0x83,0xf9,0x7b
+
+# ATT:   ccmpbew {dfv=of} $1234, %dx
+# INTEL: ccmpbe {dfv=of} dx, 1234
+0x62,0xf4,0x45,0x06,0x81,0xfa,0xd2,0x04
+
+# ATT:   ccmpbel {dfv=of} $123456, %ecx
+# INTEL: ccmpbe {dfv=of} ecx, 123456
+0x62,0xf4,0x44,0x06,0x81,0xf9,0x40,0xe2,0x01,0x00
+
+# ATT:   ccmpbeq {dfv=of} $123456, %r9
+# INTEL: ccmpbe {dfv=of} r9, 123456
+0x62,0xd4,0xc4,0x06,0x81,0xf9,0x40,0xe2,0x01,0x00
+
+# ATT:   ccmpbeb {dfv=of} %bl, %dl
+# INTEL: ccmpbe {dfv=of} dl, bl
+0x62,0xf4,0x44,0x06,0x3a,0xd3
+
+# ATT:   ccmpbew {dfv=of} %dx, %ax
+# INTEL: ccmpbe {dfv=of} ax, dx
+0x62,0xf4,0x45,0x06,0x3b,0xc2
+
+# ATT:   ccmpbel {dfv=of} %ecx, %edx
+# INTEL: ccmpbe {dfv=of} edx, ecx
+0x62,0xf4,0x44,0x06,0x3b,0xd1
+
+# ATT:   ccmpbeq {dfv=of} %r9, %r15
+# INTEL: ccmpbe {dfv=of} r15, r9
+0x62,0x54,0xc4,0x06,0x3b,0xf9
+
 # ATT:   ccmpob {dfv=of} $123, 123(%r8,%rax,4)
 # INTEL: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123
 0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b
diff --git a/llvm/test/MC/X86/apx/ccmp-att.s b/llvm/test/MC/X86/apx/ccmp-att.s
index 32e6f371600bcd7..e7dd91d7da66ccb 100644
--- a/llvm/test/MC/X86/apx/ccmp-att.s
+++ b/llvm/test/MC/X86/apx/ccmp-att.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s
 # RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR
 
-# ERROR-COUNT-428: error:
+# ERROR-COUNT-454: error:
 # ERROR-NOT: error:
 ## Condition flags
 
@@ -899,6 +899,84 @@
 # CHECK: ccmpneq {dfv=of} %r9, %r15
 # CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0xcf]
          ccmpneq {dfv=of} %r9, %r15
+# CHECK: ccmpbeb {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x80,0x7c,0x80,0x7b,0x7b]
+         ccmpbeb {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: ccmpbew {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x83,0x7c,0x80,0x7b,0x7b]
+         ccmpbew {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: ccmpbew {dfv=of} $1234, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x81,0x7c,0x80,0x7b,0xd2,0x04]
+         ccmpbew {dfv=of} $1234, 123(%r8,%rax,4)
+# CHECK: ccmpbel {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x83,0x7c,0x80,0x7b,0x7b]
+         ccmpbel {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: ccmpbel {dfv=of} $123456, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00]
+         ccmpbel {dfv=of} $123456, 123(%r8,%rax,4)
+# CHECK: ccmpbeq {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x83,0x7c,0x80,0x7b,0x7b]
+         ccmpbeq {dfv=of} $123, 123(%r8,%rax,4)
+# CHECK: ccmpbeq {dfv=of} $123456, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00]
+         ccmpbeq {dfv=of} $123456, 123(%r8,%rax,4)
+# CHECK: ccmpbeb {dfv=of} %bl, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x38,0x5c,0x80,0x7b]
+         ccmpbeb {dfv=of} %bl, 123(%r8,%rax,4)
+# CHECK: ccmpbew {dfv=of} %dx, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x39,0x54,0x80,0x7b]
+         ccmpbew {dfv=of} %dx, 123(%r8,%rax,4)
+# CHECK: ccmpbel {dfv=of} %ecx, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x39,0x4c,0x80,0x7b]
+         ccmpbel {dfv=of} %ecx, 123(%r8,%rax,4)
+# CHECK: ccmpbeq {dfv=of} %r9, 123(%r8,%rax,4)
+# CHECK: encoding: [0x62,0x54,0xc4,0x06,0x39,0x4c,0x80,0x7b]
+         ccmpbeq {dfv=of} %r9, 123(%r8,%rax,4)
+# CHECK: ccmpbeb {dfv=of} 123(%r8,%rax,4), %bl
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x3a,0x5c,0x80,0x7b]
+         ccmpbeb {dfv=of} 123(%r8,%rax,4), %bl
+# CHECK: ccmpbew {dfv=of} 123(%r8,%rax,4), %dx
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x3b,0x54,0x80,0x7b]
+         ccmpbew {dfv=of} 123(%r8,%rax,4), %dx
+# CHECK: ccmpbel {dfv=of} 123(%r8,%rax,4), %ecx
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x3b,0x4c,0x80,0x7b]
+         ccmpbel {dfv=of} 123(%r8,%rax,4), %ecx
+# CHECK: ccmpbeq {dfv=of} 123(%r8,%rax,4), %r9
+# CHECK: encoding: [0x62,0x54,0xc4,0x06,0x3b,0x4c,0x80,0x7b]
+         ccmpbeq {dfv=of} 123(%r8,%rax,4), %r9
+# CHECK: ccmpbeb {dfv=of} $123, %bl
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x80,0xfb,0x7b]
+         ccmpbeb {dfv=of} $123, %bl
+# CHECK: ccmpbew {dfv=of} $123, %dx
+# CHECK: encoding: [0x62,0xf4,0x45,0x06,0x83,0xfa,0x7b]
+         ccmpbew {dfv=of} $123, %dx
+# CHECK: ccmpbel {dfv=of} $123, %ecx
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x83,0xf9,0x7b]
+         ccmpbel {dfv=of} $123, %ecx
+# CHECK: ccmpbeq {dfv=of} $123, %r9
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x83,0xf9,0x7b]
+         ccmpbeq {dfv=of} $123, %r9
+# CHECK: ccmpbew {dfv=of} $1234, %dx
+# CHECK: encoding: [0x62,0xf4,0x45,0x06,0x81,0xfa,0xd2,0x04]
+         ccmpbew {dfv=of} $1234, %dx
+# CHECK: ccmpbel {dfv=of} $123456, %ecx
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x81,0xf9,0x40,0xe2,0x01,0x00]
+         ccmpbel {dfv=of} $123456, %ecx
+# CHECK: ccmpbeq {dfv=of} $123456, %r9
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x81,0xf9,0x40,0xe2,0x01,0x00]
+         ccmpbeq {dfv=of} $123456, %r9
+# CHECK: ccmpbeb {dfv=of} %bl, %dl
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x38,0xda]
+         ccmpbeb {dfv=of} %bl, %dl
+# CHECK: ccmpbew {dfv=of} %dx, %ax
+# CHECK: encoding: [0x62,0xf4,0x45,0x06,0x39,0xd0]
+         ccmpbew {dfv=of} %dx, %ax
+# CHECK: ccmpbel {dfv=of} %ecx, %edx
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x39,0xca]
+         ccmpbel {dfv=of} %ecx, %edx
+# CHECK: ccmpbeq {dfv=of} %r9, %r15
+# CHECK: encoding: [0x62,0x54,0xc4,0x06,0x39,0xcf]
+         ccmpbeq {dfv=of} %r9, %r15
 # CHECK: ccmpob {dfv=of} $123, 123(%r8,%rax,4)
 # CHECK: encoding: [0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b]
          ccmpob {dfv=of} $123, 123(%r8,%rax,4)
diff --git a/llvm/test/MC/X86/apx/ccmp-intel.s b/llvm/test/MC/X86/apx/ccmp-intel.s
index 5e64b63228cb855..ec3f72b8a0a8d13 100644
--- a/llvm/test/MC/X86/apx/ccmp-intel.s
+++ b/llvm/test/MC/X86/apx/ccmp-intel.s
@@ -896,6 +896,84 @@
 # CHECK: ccmpne {dfv=of} r15, r9
 # CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0xcf]
          ccmpne {dfv=of} r15, r9
+# CHECK: ccmpbe {dfv=of} byte ptr [r8 + 4*rax + 123], 123
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x80,0x7c,0x80,0x7b,0x7b]
+         ccmpbe {dfv=of} byte ptr [r8 + 4*rax + 123], 123
+# CHECK: ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], 123
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x83,0x7c,0x80,0x7b,0x7b]
+         ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], 123
+# CHECK: ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], 1234
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x81,0x7c,0x80,0x7b,0xd2,0x04]
+         ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], 1234
+# CHECK: ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], 123
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x83,0x7c,0x80,0x7b,0x7b]
+         ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], 123
+# CHECK: ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00]
+         ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456
+# CHECK: ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], 123
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x83,0x7c,0x80,0x7b,0x7b]
+         ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], 123
+# CHECK: ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00]
+         ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456
+# CHECK: ccmpbe {dfv=of} byte ptr [r8 + 4*rax + 123], bl
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x38,0x5c,0x80,0x7b]
+         ccmpbe {dfv=of} byte ptr [r8 + 4*rax + 123], bl
+# CHECK: ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], dx
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x39,0x54,0x80,0x7b]
+         ccmpbe {dfv=of} word ptr [r8 + 4*rax + 123], dx
+# CHECK: ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x39,0x4c,0x80,0x7b]
+         ccmpbe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx
+# CHECK: ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], r9
+# CHECK: encoding: [0x62,0x54,0xc4,0x06,0x39,0x4c,0x80,0x7b]
+         ccmpbe {dfv=of} qword ptr [r8 + 4*rax + 123], r9
+# CHECK: ccmpbe {dfv=of} bl, byte ptr [r8 + 4*rax + 123]
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x3a,0x5c,0x80,0x7b]
+         ccmpbe {dfv=of} bl, byte ptr [r8 + 4*rax + 123]
+# CHECK: ccmpbe {dfv=of} dx, word ptr [r8 + 4*rax + 123]
+# CHECK: encoding: [0x62,0xd4,0x45,0x06,0x3b,0x54,0x80,0x7b]
+         ccmpbe {dfv=of} dx, word ptr [r8 + 4*rax + 123]
+# CHECK: ccmpbe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123]
+# CHECK: encoding: [0x62,0xd4,0x44,0x06,0x3b,0x4c,0x80,0x7b]
+         ccmpbe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123]
+# CHECK: ccmpbe {dfv=of} r9, qword ptr [r8 + 4*rax + 123]
+# CHECK: encoding: [0x62,0x54,0xc4,0x06,0x3b,0x4c,0x80,0x7b]
+         ccmpbe {dfv=of} r9, qword ptr [r8 + 4*rax + 123]
+# CHECK: ccmpbe {dfv=of} bl, 123
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x80,0xfb,0x7b]
+         ccmpbe {dfv=of} bl, 123
+# CHECK: ccmpbe {dfv=of} dx, 123
+# CHECK: encoding: [0x62,0xf4,0x45,0x06,0x83,0xfa,0x7b]
+         ccmpbe {dfv=of} dx, 123
+# CHECK: ccmpbe {dfv=of} ecx, 123
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x83,0xf9,0x7b]
+         ccmpbe {dfv=of} ecx, 123
+# CHECK: ccmpbe {dfv=of} r9, 123
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x83,0xf9,0x7b]
+         ccmpbe {dfv=of} r9, 123
+# CHECK: ccmpbe {dfv=of} dx, 1234
+# CHECK: encoding: [0x62,0xf4,0x45,0x06,0x81,0xfa,0xd2,0x04]
+         ccmpbe {dfv=of} dx, 1234
+# CHECK: ccmpbe {dfv=of} ecx, 123456
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x81,0xf9,0x40,0xe2,0x01,0x00]
+         ccmpbe {dfv=of} ecx, 123456
+# CHECK: ccmpbe {dfv=of} r9, 123456
+# CHECK: encoding: [0x62,0xd4,0xc4,0x06,0x81,0xf9,0x40,0xe2,0x01,0x00]
+         ccmpbe {dfv=of} r9, 123456
+# CHECK: ccmpbe {dfv=of} dl, bl
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x38,0xda]
+         ccmpbe {dfv=of} dl, bl
+# CHECK: ccmpbe {dfv=of} ax, dx
+# CHECK: encoding: [0x62,0xf4,0x45,0x06,0x39,0xd0]
+         ccmpbe {dfv=of} ax, dx
+# CHECK: ccmpbe {dfv=of} edx, ecx
+# CHECK: encoding: [0x62,0xf4,0x44,0x06,0x39,0xca]
+         ccmpbe {dfv=of} edx, ecx
+# CHECK: ccmpbe {dfv=of} r15, r9
+# CHECK: encoding: [0x62,0x54,0xc4,0x06,0x39,0xcf]
+         ccmpbe {dfv=of} r15, r9
 # CHECK: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123
 # CHECK: encoding: [0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b]
          ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll
index 24d624c221f46bd..09e2c53465cd730 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll
@@ -6,123 +6,165 @@
 
 target triple = "aarch64-linux-gnu"
 
-define { <16 x i8>, <16 x i8> } @deinterleave_i8_factor2(ptr %ptr) {
-; NEON-LABEL: define { <16 x i8>, <16 x i8> } @deinterleave_i8_factor2
+define void @deinterleave_i8_factor2(ptr %ptr) {
+; NEON-LABEL: define void @deinterleave_i8_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[LDN:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2.v16i8.p0(ptr [[PTR]])
-; NEON-NEXT:    ret { <16 x i8>, <16 x i8> } [[LDN]]
+; NEON-NEXT:    [[TMP1:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[LDN]], 0
+; NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[LDN]], 1
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <16 x i8>, <16 x i8> } @deinterleave_i8_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_i8_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <32 x i8>, ptr [[PTR]], align 1
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <16 x i8>, <16 x i8> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <32 x i8>, ptr %ptr, align 1
   %deinterleave = tail call { <16 x i8>, <16 x i8> } @llvm.vector.deinterleave2.v32i8(<32 x i8> %load)
-  ret { <16 x i8>, <16 x i8> } %deinterleave
+  %extract1 = extractvalue { <16 x i8>, <16 x i8> } %deinterleave, 0
+  %extract2 = extractvalue { <16 x i8>, <16 x i8> } %deinterleave, 1
+  ret void
 }
 
-define { <8 x i16>, <8 x i16> } @deinterleave_i16_factor2(ptr %ptr) {
-; NEON-LABEL: define { <8 x i16>, <8 x i16> } @deinterleave_i16_factor2
+define void @deinterleave_i16_factor2(ptr %ptr) {
+; NEON-LABEL: define void @deinterleave_i16_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[LDN:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2.v8i16.p0(ptr [[PTR]])
-; NEON-NEXT:    ret { <8 x i16>, <8 x i16> } [[LDN]]
+; NEON-NEXT:    [[TMP1:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN]], 0
+; NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN]], 1
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <8 x i16>, <8 x i16> } @deinterleave_i16_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_i16_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <16 x i16>, ptr [[PTR]], align 2
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <8 x i16>, <8 x i16> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <16 x i16>, ptr %ptr, align 2
   %deinterleave = tail call { <8 x i16>, <8 x i16> } @llvm.vector.deinterleave2.v16i16(<16 x i16> %load)
-  ret { <8 x i16>, <8 x i16> } %deinterleave
+  %extract1 = extractvalue { <8 x i16>, <8 x i16> } %deinterleave, 0
+  %extract2 = extractvalue { <8 x i16>, <8 x i16> } %deinterleave, 1
+  ret void
 }
 
-define { <4 x i32>, <4 x i32> } @deinterleave_8xi32_factor2(ptr %ptr) {
-; NEON-LABEL: define { <4 x i32>, <4 x i32> } @deinterleave_8xi32_factor2
+define void @deinterleave_8xi32_factor2(ptr %ptr) {
+; NEON-LABEL: define void @deinterleave_8xi32_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr [[PTR]])
-; NEON-NEXT:    ret { <4 x i32>, <4 x i32> } [[LDN]]
+; NEON-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <4 x i32>, <4 x i32> } @deinterleave_8xi32_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_8xi32_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <8 x i32>, ptr [[PTR]], align 4
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <4 x i32>, <4 x i32> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <8 x i32>, ptr %ptr, align 4
   %deinterleave = tail call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %load)
-  ret { <4 x i32>, <4 x i32> } %deinterleave
+  %extract1 = extractvalue { <4 x i32>, <4 x i32> } %deinterleave, 0
+  %extract2 = extractvalue { <4 x i32>, <4 x i32> } %deinterleave, 1
+  ret void
 }
 
-define { <2 x i64>, <2 x i64> } @deinterleave_i64_factor2(ptr %ptr) {
-; NEON-LABEL: define { <2 x i64>, <2 x i64> } @deinterleave_i64_factor2
+define void @deinterleave_i64_factor2(ptr %ptr) {
+; NEON-LABEL: define void @deinterleave_i64_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[PTR]])
-; NEON-NEXT:    ret { <2 x i64>, <2 x i64> } [[LDN]]
+; NEON-NEXT:    [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
+; NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <2 x i64>, <2 x i64> } @deinterleave_i64_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_i64_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <4 x i64>, ptr [[PTR]], align 8
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <2 x i64>, <2 x i64> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <4 x i64>, ptr %ptr, align 8
   %deinterleave = tail call { <2 x i64>, <2 x i64> } @llvm.vector.deinterleave2.v4i64(<4 x i64> %load)
-  ret { <2 x i64>, <2 x i64> } %deinterleave
+  %extract1 = extractvalue { <2 x i64>, <2 x i64> } %deinterleave, 0
+  %extract2 = extractvalue { <2 x i64>, <2 x i64> } %deinterleave, 1
+  ret void
 }
 
-define { <4 x float>, <4 x float> } @deinterleave_float_factor2(ptr %ptr) {
-; NEON-LABEL: define { <4 x float>, <4 x float> } @deinterleave_float_factor2
+define void @deinterleave_float_factor2(ptr %ptr) {
+; NEON-LABEL: define void @deinterleave_float_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[LDN:%.*]] = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr [[PTR]])
-; NEON-NEXT:    ret { <4 x float>, <4 x float> } [[LDN]]
+; NEON-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 0
+; NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[LDN]], 1
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <4 x float>, <4 x float> } @deinterleave_float_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_float_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <8 x float>, ptr [[PTR]], align 4
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <4 x float>, <4 x float> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <4 x float>, <4 x float> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <8 x float>, ptr %ptr, align 4
   %deinterleave = tail call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %load)
-  ret { <4 x float>, <4 x float> } %deinterleave
+  %extract1 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
+  %extract2 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+  ret void
 }
 
-define { <2 x double>, <2 x double> } @deinterleave_double_factor2(ptr %ptr) {
-; NEON-LABEL: define { <2 x double>, <2 x double> } @deinterleave_double_factor2
+define void @deinterleave_double_factor2(ptr %ptr) {
+; NEON-LABEL: define void @deinterleave_double_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[LDN:%.*]] = call { <2 x double>, <2 x double> } @llvm.aarch64.neon.ld2.v2f64.p0(ptr [[PTR]])
-; NEON-NEXT:    ret { <2 x double>, <2 x double> } [[LDN]]
+; NEON-NEXT:    [[TMP1:%.*]] = extractvalue { <2 x double>, <2 x double> } [[LDN]], 0
+; NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x double>, <2 x double> } [[LDN]], 1
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <2 x double>, <2 x double> } @deinterleave_double_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_double_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <4 x double>, ptr [[PTR]], align 8
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <2 x double>, <2 x double> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <2 x double>, <2 x double> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <2 x double>, <2 x double> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <4 x double>, ptr %ptr, align 8
   %deinterleave = tail call { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double> %load)
-  ret { <2 x double>, <2 x double> } %deinterleave
+  %extract1 = extractvalue { <2 x double>, <2 x double> } %deinterleave, 0
+  %extract2 = extractvalue { <2 x double>, <2 x double> } %deinterleave, 1
+  ret void
 }
 
-define { <2 x ptr>, <2 x ptr> } @deinterleave_ptr_factor2(ptr %ptr) {
-; NEON-LABEL: define { <2 x ptr>, <2 x ptr> } @deinterleave_ptr_factor2
+define void @deinterleave_ptr_factor2(ptr %ptr) {
+; NEON-LABEL: define void @deinterleave_ptr_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[LDN:%.*]] = call { <2 x ptr>, <2 x ptr> } @llvm.aarch64.neon.ld2.v2p0.p0(ptr [[PTR]])
-; NEON-NEXT:    ret { <2 x ptr>, <2 x ptr> } [[LDN]]
+; NEON-NEXT:    [[TMP1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[LDN]], 0
+; NEON-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[LDN]], 1
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <2 x ptr>, <2 x ptr> } @deinterleave_ptr_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_ptr_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <4 x ptr>, ptr [[PTR]], align 8
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave2.v4p0(<4 x ptr> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <2 x ptr>, <2 x ptr> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <4 x ptr>, ptr %ptr, align 8
   %deinterleave = tail call { <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave2.v4p0(<4 x ptr> %load)
-  ret { <2 x ptr>, <2 x ptr> } %deinterleave
+  %extract1 = extractvalue { <2 x ptr>, <2 x ptr> } %deinterleave, 0
+  %extract2 = extractvalue { <2 x ptr>, <2 x ptr> } %deinterleave, 1
+  ret void
 }
 
 define void @interleave_i8_factor2(ptr %ptr, <16 x i8> %l, <16 x i8> %r) {
@@ -244,8 +286,8 @@ define void @interleave_ptr_factor2(ptr %ptr, <2 x ptr> %l, <2 x ptr> %r) {
   ret void
 }
 
-define { <16 x i16>, <16 x i16> } @deinterleave_wide_i16_factor2(ptr %ptr) #0 {
-; NEON-LABEL: define { <16 x i16>, <16 x i16> } @deinterleave_wide_i16_factor2
+define void @deinterleave_wide_i16_factor2(ptr %ptr) #0 {
+; NEON-LABEL: define void @deinterleave_wide_i16_factor2
 ; NEON-SAME: (ptr [[PTR:%.*]]) {
 ; NEON-NEXT:    [[TMP1:%.*]] = getelementptr <8 x i16>, ptr [[PTR]], i64 0
 ; NEON-NEXT:    [[LDN:%.*]] = call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2.v8i16.p0(ptr [[TMP1]])
@@ -259,19 +301,21 @@ define { <16 x i16>, <16 x i16> } @deinterleave_wide_i16_factor2(ptr %ptr) #0 {
 ; NEON-NEXT:    [[TMP8:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v8i16(<16 x i16> [[TMP3]], <8 x i16> [[TMP7]], i64 8)
 ; NEON-NEXT:    [[TMP9:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[LDN1]], 1
 ; NEON-NEXT:    [[TMP10:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v8i16(<16 x i16> [[TMP5]], <8 x i16> [[TMP9]], i64 8)
-; NEON-NEXT:    [[TMP11:%.*]] = insertvalue { <16 x i16>, <16 x i16> } poison, <16 x i16> [[TMP8]], 0
-; NEON-NEXT:    [[TMP12:%.*]] = insertvalue { <16 x i16>, <16 x i16> } [[TMP11]], <16 x i16> [[TMP10]], 1
-; NEON-NEXT:    ret { <16 x i16>, <16 x i16> } [[TMP12]]
+; NEON-NEXT:    ret void
 ;
-; SVE-FIXED-LABEL: define { <16 x i16>, <16 x i16> } @deinterleave_wide_i16_factor2
+; SVE-FIXED-LABEL: define void @deinterleave_wide_i16_factor2
 ; SVE-FIXED-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; SVE-FIXED-NEXT:    [[LOAD:%.*]] = load <32 x i16>, ptr [[PTR]], align 2
 ; SVE-FIXED-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <16 x i16>, <16 x i16> } @llvm.vector.deinterleave2.v32i16(<32 x i16> [[LOAD]])
-; SVE-FIXED-NEXT:    ret { <16 x i16>, <16 x i16> } [[DEINTERLEAVE]]
+; SVE-FIXED-NEXT:    [[EXTRACT1:%.*]] = extractvalue { <16 x i16>, <16 x i16> } [[DEINTERLEAVE]], 0
+; SVE-FIXED-NEXT:    [[EXTRACT2:%.*]] = extractvalue { <16 x i16>, <16 x i16> } [[DEINTERLEAVE]], 1
+; SVE-FIXED-NEXT:    ret void
 ;
   %load = load <32 x i16>, ptr %ptr, align 2
   %deinterleave = tail call { <16 x i16>, <16 x i16> } @llvm.vector.deinterleave2.v32i16(<32 x i16> %load)
-  ret { <16 x i16>, <16 x i16> } %deinterleave
+  %extract1 = extractvalue { <16 x i16>, <16 x i16> } %deinterleave, 0
+  %extract2 = extractvalue { <16 x i16>, <16 x i16> } %deinterleave, 1
+  ret void
 }
 
 define void @interleave_wide_ptr_factor2(ptr %ptr, <8 x ptr> %l, <8 x ptr> %r) {
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll
index 2a05718cc4161c8..e5b56eb54f927d4 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll
@@ -4,81 +4,109 @@
 
 target triple = "aarch64-linux-gnu"
 
-define { <vscale x 16 x i8>, <vscale x 16 x i8> } @deinterleave_nxi8_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 16 x i8>, <vscale x 16 x i8> } @deinterleave_nxi8_factor2
+define void @deinterleave_nxi8_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi8_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> shufflevector (<vscale x 16 x i1> insertelement (<vscale x 16 x i1> poison, i1 true, i64 0), <vscale x 16 x i1> poison, <vscale x 16 x i32> zeroinitializer), ptr [[PTR]])
-; CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8> } [[LDN]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[LDN]], 1
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 32 x i8>, ptr %ptr, align 1
   %deinterleave = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %load)
-  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave
+  %extract1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 1
+  %extract2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleave, 0
+  ret void
 }
 
-define { <vscale x 8 x i16>, <vscale x 8 x i16> } @deinterleave_nxi16_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 8 x i16>, <vscale x 8 x i16> } @deinterleave_nxi16_factor2
+define void @deinterleave_nxi16_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi16_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer), ptr [[PTR]])
-; CHECK-NEXT:    ret { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 1
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 16 x i16>, ptr %ptr, align 2
   %deinterleave = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> %load)
-  ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %deinterleave
+  %extract1 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %deinterleave, 1
+  ret void
 }
 
-define { <vscale x 4 x i32>, <vscale x 4 x i32> } @deinterleave_nx8xi32_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 4 x i32>, <vscale x 4 x i32> } @deinterleave_nx8xi32_factor2
+define void @deinterleave_nx8xi32_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_nx8xi32_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[PTR]])
-; CHECK-NEXT:    ret { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 8 x i32>, ptr %ptr, align 4
   %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %load)
-  ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave
+  %extract1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 1
+  ret void
 }
 
-define { <vscale x 2 x i64>, <vscale x 2 x i64> } @deinterleave_nxi64_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 2 x i64>, <vscale x 2 x i64> } @deinterleave_nxi64_factor2
+define void @deinterleave_nxi64_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_nxi64_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), ptr [[PTR]])
-; CHECK-NEXT:    ret { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 4 x i64>, ptr %ptr, align 8
   %deinterleave = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> %load)
-  ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %deinterleave
+  %extract1 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %deinterleave, 1
+  ret void
 }
 
-define { <vscale x 4 x float>, <vscale x 4 x float> } @deinterleave_nxfloat_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 4 x float>, <vscale x 4 x float> } @deinterleave_nxfloat_factor2
+define void @deinterleave_nxfloat_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_nxfloat_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2.sret.nxv4f32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[PTR]])
-; CHECK-NEXT:    ret { <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 1
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 8 x float>, ptr %ptr, align 4
   %deinterleave = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %load)
-  ret { <vscale x 4 x float>, <vscale x 4 x float> } %deinterleave
+  %extract1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %deinterleave, 1
+  ret void
 }
 
-define { <vscale x 2 x double>, <vscale x 2 x double> } @deinterleave_nxdouble_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 2 x double>, <vscale x 2 x double> } @deinterleave_nxdouble_factor2
+define void @deinterleave_nxdouble_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_nxdouble_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), ptr [[PTR]])
-; CHECK-NEXT:    ret { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 1
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 4 x double>, ptr %ptr, align 8
   %deinterleave = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %load)
-  ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleave
+  %extract1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleave, 1
+  ret void
 }
 
-define { <vscale x 2 x ptr>, <vscale x 2 x ptr> } @deinterleave_nxptr_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 2 x ptr>, <vscale x 2 x ptr> } @deinterleave_nxptr_factor2
+define void @deinterleave_nxptr_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_nxptr_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x ptr>, <vscale x 2 x ptr> } @llvm.aarch64.sve.ld2.sret.nxv2p0(<vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), ptr [[PTR]])
-; CHECK-NEXT:    ret { <vscale x 2 x ptr>, <vscale x 2 x ptr> } [[LDN]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x ptr>, <vscale x 2 x ptr> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x ptr>, <vscale x 2 x ptr> } [[LDN]], 1
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 4 x ptr>, ptr %ptr, align 8
   %deinterleave = tail call { <vscale x 2 x ptr>, <vscale x 2 x ptr> } @llvm.vector.deinterleave2.nxv4p0(<vscale x 4 x ptr> %load)
-  ret { <vscale x 2 x ptr>, <vscale x 2 x ptr> } %deinterleave
+  %extract1 = extractvalue { <vscale x 2 x ptr>, <vscale x 2 x ptr> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 2 x ptr>, <vscale x 2 x ptr> } %deinterleave, 1
+  ret void
 }
 
 define void @interleave_nxi8_factor2(ptr %ptr, <vscale x 16 x i8> %l, <vscale x 16 x i8> %r) #0 {
@@ -160,8 +188,8 @@ define void @interleave_nxptr_factor2(ptr %ptr, <vscale x 2 x ptr> %l, <vscale x
 
 ;;; Check that we 'legalize' operations that are wider than the target supports.
 
-define { <vscale x 16 x i32>, <vscale x 16 x i32> } @deinterleave_wide_nxi32_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 16 x i32>, <vscale x 16 x i32> } @deinterleave_wide_nxi32_factor2
+define void @deinterleave_wide_nxi32_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_wide_nxi32_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[PTR]], i64 0
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[TMP1]])
@@ -187,17 +215,17 @@ define { <vscale x 16 x i32>, <vscale x 16 x i32> } @deinterleave_wide_nxi32_fac
 ; CHECK-NEXT:    [[TMP18:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP13]], <vscale x 4 x i32> [[TMP17]], i64 12)
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN3]], 1
 ; CHECK-NEXT:    [[TMP20:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP15]], <vscale x 4 x i32> [[TMP19]], i64 12)
-; CHECK-NEXT:    [[TMP21:%.*]] = insertvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } poison, <vscale x 16 x i32> [[TMP18]], 0
-; CHECK-NEXT:    [[TMP22:%.*]] = insertvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } [[TMP21]], <vscale x 16 x i32> [[TMP20]], 1
-; CHECK-NEXT:    ret { <vscale x 16 x i32>, <vscale x 16 x i32> } [[TMP22]]
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 32 x i32>, ptr %ptr, align 4
   %deinterleave = tail call { <vscale x 16 x i32>, <vscale x 16 x i32> } @llvm.vector.deinterleave2.nxv32i32(<vscale x 32 x i32> %load)
-  ret { <vscale x 16 x i32>, <vscale x 16 x i32> } %deinterleave
+  %extract1 = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } %deinterleave, 1
+  ret void
 }
 
-define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_wide_nxdouble_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_wide_nxdouble_factor2
+define void @deinterleave_wide_nxdouble_factor2(ptr %ptr) #0 {
+; CHECK-LABEL: define void @deinterleave_wide_nxdouble_factor2
 ; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[PTR]], i64 0
 ; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), ptr [[TMP1]])
@@ -211,13 +239,13 @@ define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_wide_nxdou
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP7]], i64 2)
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN1]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP5]], <vscale x 2 x double> [[TMP9]], i64 2)
-; CHECK-NEXT:    [[TMP11:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } poison, <vscale x 4 x double> [[TMP8]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP11]], <vscale x 4 x double> [[TMP10]], 1
-; CHECK-NEXT:    ret { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP12]]
+; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 8 x double>, ptr %ptr, align 8
   %deinterleave = tail call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %load)
-  ret { <vscale x 4 x double>, <vscale x 4 x double> } %deinterleave
+  %extract1 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %deinterleave, 0
+  %extract2 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %deinterleave, 1
+  ret void
 }
 
 define void @interleave_wide_nxdouble_factor2(ptr %ptr, <vscale x 4 x double> %l, <vscale x 4 x double> %r) #0 {
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave4.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave4.ll
new file mode 100644
index 000000000000000..06ecff672988135
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave4.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -S | FileCheck %s
+
+
+define void @deinterleave4(ptr %src) {
+; CHECK-LABEL: define void @deinterleave4
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 3
+; CHECK-NEXT:    [[SUM:%.*]] = add <vscale x 4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub <vscale x 4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    ret void
+;
+
+  %load = load <vscale x 16 x i32>, ptr %src, align 4
+  %deinterleave_src = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %load)
+  %3 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 0
+  %4 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 1
+  %deinterleave_half1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %3)
+  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 0
+  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 1
+  %deinterleave_half2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %4)
+  %7 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 0
+  %8 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 1
+  %sum = add <vscale x 4 x i32> %5, %7
+  %sub = sub <vscale x 4 x i32> %6, %8
+  ret void
+}
+
+define void @wide_deinterleave4(ptr %src) {
+; CHECK-LABEL: define void @wide_deinterleave4
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[SRC]], i64 0
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[SRC]], i64 4
+; CHECK-NEXT:    [[LDN1:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[TMP10]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP11]], i64 4)
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP13]], i64 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 2
+; CHECK-NEXT:    [[TMP16:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP15]], i64 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP9]], <vscale x 4 x i32> [[TMP17]], i64 4)
+; CHECK-NEXT:    [[SUM:%.*]] = add <vscale x 8 x i32> [[TMP12]], [[TMP14]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub <vscale x 8 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    ret void
+;
+  %load = load <vscale x 32 x i32>, ptr %src, align 4
+  %deinterleave_src = tail call { <vscale x 16 x i32>, <vscale x 16 x i32> } @llvm.vector.deinterleave2.nxv32i32(<vscale x 32 x i32> %load)
+  %3 = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } %deinterleave_src, 0
+  %4 = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } %deinterleave_src, 1
+  %deinterleave_half1 = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %3)
+  %5 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half1, 0
+  %6 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half1, 1
+  %deinterleave_half2 = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %4)
+  %7 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half2, 0
+  %8 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half2, 1
+  %sum = add <vscale x 8 x i32> %5, %7
+  %sub = sub <vscale x 8 x i32> %6, %8
+  ret void
+}
+
+define void @mix_deinterleave4_deinterleave2(ptr %src) {
+; CHECK-LABEL: define void @mix_deinterleave4_deinterleave2
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[SRC]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 3
+; CHECK-NEXT:    [[LDN1:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[SRC]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 1
+; CHECK-NEXT:    ret void
+;
+
+  %load = load <vscale x 16 x i32>, ptr %src, align 4
+  %deinterleave_src = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %load)
+  %3 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 0
+  %4 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 1
+  %deinterleave_half1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %3)
+  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 0
+  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 1
+  %deinterleave_half2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %4)
+  %7 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 0
+  %8 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 1
+
+  %load2 = load <vscale x 8 x i32>, ptr %src, align 4
+  %deinterleave_src2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 8 x i32> %load2)
+  %ld2_1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_src2, 0
+  %ld2_2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_src2, 1
+  ret void
+}
+
+define void @negative_deinterleave4_test(ptr %src) {
+; CHECK-LABEL: define void @negative_deinterleave4_test
+; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[SRC]], i64 0
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[SRC]], i64 2
+; CHECK-NEXT:    [[LDN1:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP7]], i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP9]], i64 4)
+; CHECK-NEXT:    [[DEINTERLEAVE_HALF1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP8]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE_HALF1]], 0
+; CHECK-NEXT:    [[DEINTERLEAVE_HALF2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP10]])
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE_HALF2]], 1
+; CHECK-NEXT:    ret void
+;
+  %load = load <vscale x 16 x i32>, ptr %src, align 4
+  %deinterleave_src = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %load)
+  %3 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 0
+  %4 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 1
+  %deinterleave_half1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %3)
+  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 0
+  %deinterleave_half2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %4)
+  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 1
+
+  ret void
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave4.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave4.ll
new file mode 100644
index 000000000000000..ba9bff093678c2c
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave4.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -S | FileCheck %s
+
+
+define void @interleave4(ptr %dst, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d) {
+; CHECK-LABEL: define void @interleave4
+; CHECK-SAME: (ptr [[DST:%.*]], <vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i32> [[B:%.*]], <vscale x 4 x i32> [[C:%.*]], <vscale x 4 x i32> [[D:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[A]], <vscale x 4 x i32> [[B]], <vscale x 4 x i32> [[C]], <vscale x 4 x i32> [[D]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[DST]])
+; CHECK-NEXT:    ret void
+;
+  %interleaved.half1 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %c)
+  %interleaved.half2 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %b, <vscale x 4 x i32> %d)
+  %interleaved.vec = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %interleaved.half1, <vscale x 8 x i32> %interleaved.half2)
+  store <vscale x 16 x i32> %interleaved.vec, ptr %dst, align 4
+  ret void
+}
+
+define void @wide_interleave4(ptr %dst, <vscale x 8 x i32> %a, <vscale x 8 x i32> %b, <vscale x 8 x i32> %c, <vscale x 8 x i32> %d) {
+; CHECK-LABEL: define void @wide_interleave4
+; CHECK-SAME: (ptr [[DST:%.*]], <vscale x 8 x i32> [[A:%.*]], <vscale x 8 x i32> [[B:%.*]], <vscale x 8 x i32> [[C:%.*]], <vscale x 8 x i32> [[D:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[DST]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[A]], i64 0)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[B]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[C]], i64 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[D]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[DST]], i64 4
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[A]], i64 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[B]], i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[C]], i64 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[D]], i64 4)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[TMP10]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[TMP6]])
+; CHECK-NEXT:    ret void
+;
+  %interleaved.half1 = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %c)
+  %interleaved.half2 = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %b, <vscale x 8 x i32> %d)
+  %interleaved.vec = tail call <vscale x 32 x i32> @llvm.vector.interleave2.nxv32i32(<vscale x 16 x i32> %interleaved.half1, <vscale x 16 x i32> %interleaved.half2)
+  store <vscale x 32 x i32> %interleaved.vec, ptr %dst, align 4
+  ret void
+}
+
+define void @mix_interleave4_interleave2(ptr %dst1, ptr %dst2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d) {
+; CHECK-LABEL: define void @mix_interleave4_interleave2
+; CHECK-SAME: (ptr [[DST1:%.*]], ptr [[DST2:%.*]], <vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i32> [[B:%.*]], <vscale x 4 x i32> [[C:%.*]], <vscale x 4 x i32> [[D:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[A]], <vscale x 4 x i32> [[B]], <vscale x 4 x i32> [[C]], <vscale x 4 x i32> [[D]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[DST1]])
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32> [[A]], <vscale x 4 x i32> [[C]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), ptr [[DST2]])
+; CHECK-NEXT:    ret void
+;
+  %interleaved.half1 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %c)
+  %interleaved.half2 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %b, <vscale x 4 x i32> %d)
+  %interleaved.vec = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %interleaved.half1, <vscale x 8 x i32> %interleaved.half2)
+  store <vscale x 16 x i32> %interleaved.vec, ptr %dst1, align 4
+
+  %interleaved = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %c)
+  store <vscale x 8 x i32> %interleaved, ptr %dst2, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
index 73f26814f3a4bcd..8821255a86b2f84 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
@@ -1,36 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -interleaved-access -S | FileCheck %s
 ; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
 define void @load_factor2(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_factor2(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT1:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv8i16(<vscale x 8 x i16> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT2:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv8i16(<vscale x 8 x i16> [[TMP3]], i64 0)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @load_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv8i16(<vscale x 8 x i16> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv8i16(<vscale x 8 x i16> [[TMP4]], i64 0)
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <32 x i16>, ptr %ptr, align 4
   %v0 = shufflevector <32 x i16> %interleaved.vec, <32 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14,
-                                                                                  i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+  i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %v1 = shufflevector <32 x i16> %interleaved.vec, <32 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15,
-                                                                                  i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+  i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   ret void
 }
 
 define void @load_factor3(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_factor3(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 2
-; CHECK-NEXT:       [[EXT1:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT2:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT3:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP4]], i64 0)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @load_factor3(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv4i32(<vscale x 4 x i32> [[TMP6]], i64 0)
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <24 x i32>, ptr %ptr, align 4
   %v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
   %v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> poison, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
@@ -39,18 +44,20 @@ define void @load_factor3(ptr %ptr) #0 {
 }
 
 define void @load_factor4(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_factor4(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 3
-; CHECK-NEXT:       [[EXT1:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
-; CHECK-NEXT:       [[EXT2:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT:       [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT4:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @load_factor4(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <16 x i64>, ptr %ptr, align 4
   %v0 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
   %v1 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -60,54 +67,64 @@ define void @load_factor4(ptr %ptr) #0 {
 }
 
 define void @store_factor2(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
-; CHECK-LABEL:    @store_factor2(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:       [[TMP1:%.*]] = shufflevector <16 x i16> %v0, <16 x i16> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:       [[INS1:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> [[TMP1]], i64 0)
-; CHECK-NEXT:       [[TMP2:%.*]] = shufflevector <16 x i16> %v0, <16 x i16> %v1, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:       [[INS2:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> [[TMP2]], i64 0)
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> [[INS1]], <vscale x 8 x i16> [[INS2]], <vscale x 8 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @store_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]], <16 x i16> [[V0:%.*]], <16 x i16> [[V1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[V0]], <16 x i16> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[V0]], <16 x i16> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v16i16(<vscale x 8 x i16> undef, <16 x i16> [[TMP4]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = shufflevector <16 x i16> %v0, <16 x i16> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
-                                                                               i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   store <32 x i16> %interleaved.vec, ptr %ptr, align 4
   ret void
 }
 
 define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
-; CHECK-LABEL:    @store_factor3(
-; CHECK:            [[PTRUE:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-; CHECK-NEXT:       [[TMP1:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       [[INS1:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP1]], i64 0)
-; CHECK-NEXT:       [[TMP2:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:       [[INS2:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT:       [[INS3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP3]], i64 0)
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> [[INS1]], <vscale x 4 x i32> [[INS2]], <vscale x 4 x i32> [[INS3]], <vscale x 4 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @store_factor3(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <8 x i32> [[V2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0]], <8 x i32> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v8i32(<vscale x 4 x i32> undef, <8 x i32> [[TMP6]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
-                                                                i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %s1 = shufflevector <8 x i32> %v2, <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
-                                                                   i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19,
-                                                                               i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
   store <24 x i32> %interleaved.vec, ptr %ptr, align 4
   ret void
 }
 
 define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
-; CHECK-LABEL:    @store_factor4(
-; CHECK:            [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[TMP1:%.*]] = shufflevector <8 x i64> %s0, <8 x i64> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:       [[INS1:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP1]], i64 0)
-; CHECK-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i64> %s0, <8 x i64> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       [[INS2:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i64> %s0, <8 x i64> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:       [[INS3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i64> %s0, <8 x i64> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:       [[INS4:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[INS1]], <vscale x 2 x i64> [[INS2]], <vscale x 2 x i64> [[INS3]], <vscale x 2 x i64> [[INS4]], <vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @store_factor4(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]], <4 x i64> [[V2:%.*]], <4 x i64> [[V3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x i64> [[V2]], <4 x i64> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %s0 = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s1 = shufflevector <4 x i64> %v2, <4 x i64> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %interleaved.vec = shufflevector <8 x i64> %s0, <8 x i64> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
@@ -116,16 +133,18 @@ define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2
 }
 
 define void @load_ptrvec_factor2(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_ptrvec_factor2(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT1:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TOP1:%.*]] = inttoptr <4 x i64> [[EXT1]] to <4 x ptr>
-; CHECK-NEXT:       [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT2:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TOP2:%.*]] = inttoptr <4 x i64> [[EXT2]] to <4 x ptr>
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @load_ptrvec_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr <4 x i64> [[TMP3]] to <4 x ptr>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr <4 x i64> [[TMP6]] to <4 x ptr>
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <8 x ptr>, ptr %ptr, align 4
   %v0 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %v1 = shufflevector <8 x ptr> %interleaved.vec, <8 x ptr> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -133,19 +152,21 @@ define void @load_ptrvec_factor2(ptr %ptr) #0 {
 }
 
 define void @load_ptrvec_factor3(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_ptrvec_factor3(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
-; CHECK-NEXT:       [[EXT1:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TOP1:%.*]] = inttoptr <4 x i64> [[EXT1]] to <4 x ptr>
-; CHECK-NEXT:       [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT2:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TOP2:%.*]] = inttoptr <4 x i64> [[EXT2]] to <4 x ptr>
-; CHECK-NEXT:       [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT:       [[TOP3:%.*]] = inttoptr <4 x i64> [[EXT3]] to <4 x ptr>
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @load_ptrvec_factor3(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr <4 x i64> [[TMP3]] to <4 x ptr>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr <4 x i64> [[TMP6]] to <4 x ptr>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr <4 x i64> [[TMP9]] to <4 x ptr>
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <12 x ptr>, ptr %ptr, align 4
   %v0 = shufflevector <12 x ptr> %interleaved.vec, <12 x ptr> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
   %v1 = shufflevector <12 x ptr> %interleaved.vec, <12 x ptr> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
@@ -154,22 +175,24 @@ define void @load_ptrvec_factor3(ptr %ptr) #0 {
 }
 
 define void @load_ptrvec_factor4(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_ptrvec_factor4(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 3
-; CHECK-NEXT:       [[EXT1:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TOP1:%.*]] = inttoptr <4 x i64> [[EXT1]] to <4 x ptr>
-; CHECK-NEXT:       [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
-; CHECK-NEXT:       [[EXT2:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TOP2:%.*]] = inttoptr <4 x i64> [[EXT2]] to <4 x ptr>
-; CHECK-NEXT:       [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT:       [[TOP3:%.*]] = inttoptr <4 x i64> [[EXT3]] to <4 x ptr>
-; CHECK-NEXT:       [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT4:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
-; CHECK-NEXT:       [[TOP4:%.*]] = inttoptr <4 x i64> [[EXT4]] to <4 x ptr>
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @load_ptrvec_factor4(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr <4 x i64> [[TMP3]] to <4 x ptr>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr <4 x i64> [[TMP6]] to <4 x ptr>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr <4 x i64> [[TMP9]] to <4 x ptr>
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP11]], i64 0)
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr <4 x i64> [[TMP12]] to <4 x ptr>
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <16 x ptr>, ptr %ptr, align 4
   %v0 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
   %v1 = shufflevector <16 x ptr> %interleaved.vec, <16 x ptr> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
@@ -179,34 +202,40 @@ define void @load_ptrvec_factor4(ptr %ptr) #0 {
 }
 
 define void @store_ptrvec_factor2(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1) #0 {
-; CHECK-LABEL:    @store_ptrvec_factor2(
-; CHECK-NEXT:       [[TOI1:%.*]] = ptrtoint <4 x ptr> %v0 to <4 x i64>
-; CHECK-NEXT:       [[TOI2:%.*]] = ptrtoint <4 x ptr> %v1 to <4 x i64>
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[TMP1:%.*]] = shufflevector <4 x i64> [[TOI1]], <4 x i64> [[TOI2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:       [[INS1:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP1]], i64 0)
-; CHECK-NEXT:       [[TMP2:%.*]] = shufflevector <4 x i64> [[TOI1]], <4 x i64> [[TOI2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       [[INS2:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[INS1]], <vscale x 2 x i64> [[INS2]], <vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @store_ptrvec_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x ptr> [[V0:%.*]], <4 x ptr> [[V1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <4 x ptr> [[V0]] to <4 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <4 x ptr> [[V1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> [[TMP3]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   store <8 x ptr> %interleaved.vec, ptr %ptr, align 4
   ret void
 }
 
 define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2) #0 {
-; CHECK-LABEL:    @store_ptrvec_factor3(
-; CHECK:            [[TOI1:%.*]] = ptrtoint <8 x ptr> %s0 to <8 x i64>
-; CHECK-NEXT:       [[TOI2:%.*]] = ptrtoint <8 x ptr> %s1 to <8 x i64>
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[TMP1:%.*]] = shufflevector <8 x i64> [[TOI1]], <8 x i64> [[TOI2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:       [[INS1:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP1]], i64 0)
-; CHECK-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i64> [[TOI1]], <8 x i64> [[TOI2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       [[INS2:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i64> [[TOI1]], <8 x i64> [[TOI2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:       [[INS3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> [[INS1]], <vscale x 2 x i64> [[INS2]], <vscale x 2 x i64> [[INS3]], <vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @store_ptrvec_factor3(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x ptr> [[V0:%.*]], <4 x ptr> [[V1:%.*]], <4 x ptr> [[V2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x ptr> [[V0]], <4 x ptr> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x ptr> [[V2]], <4 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <8 x ptr> [[S0]] to <8 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <8 x ptr> [[S1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> [[TMP3]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %s0 = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s1 = shufflevector <4 x ptr> %v2, <4 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
   %interleaved.vec = shufflevector <8 x ptr> %s0, <8 x ptr> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
@@ -215,45 +244,51 @@ define void @store_ptrvec_factor3(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x p
 }
 
 define void @store_ptrvec_factor4(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3) #0 {
-; CHECK-LABEL:    @store_ptrvec_factor4(
-; CHECK:            [[TOI1:%.*]] = ptrtoint <8 x ptr> %s0 to <8 x i64>
-; CHECK-NEXT:       [[TOI2:%.*]] = ptrtoint <8 x ptr> %s1 to <8 x i64>
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[TMP1:%.*]] = shufflevector <8 x i64> [[TOI1]], <8 x i64> [[TOI2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:       [[INS1:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP1]], i64 0)
-; CHECK-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i64> [[TOI1]], <8 x i64> [[TOI2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       [[INS2:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i64> [[TOI1]], <8 x i64> [[TOI2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:       [[INS3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i64> [[TOI1]], <8 x i64> [[TOI2]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:       [[INS4:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[INS1]], <vscale x 2 x i64> [[INS2]], <vscale x 2 x i64> [[INS3]], <vscale x 2 x i64> [[INS4]], <vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @store_ptrvec_factor4(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x ptr> [[V0:%.*]], <4 x ptr> [[V1:%.*]], <4 x ptr> [[V2:%.*]], <4 x ptr> [[V3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x ptr> [[V0]], <4 x ptr> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x ptr> [[V2]], <4 x ptr> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <8 x ptr> [[S0]] to <8 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <8 x ptr> [[S1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> [[TMP2]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP10]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i1> [[TMP3]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %s0 = shufflevector <4 x ptr> %v0, <4 x ptr> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s1 = shufflevector <4 x ptr> %v2, <4 x ptr> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %interleaved.vec = shufflevector <8 x ptr> %s0, <8 x ptr> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13,
-                                                                               i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
   store <16 x ptr> %interleaved.vec, ptr %ptr, align 4
   ret void
 }
 
 define void @load_factor2_wide(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_factor2_wide(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT1:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT2:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       [[TMP4:%.*]] = getelementptr i64, ptr %ptr, i32 8
-; CHECK-NEXT:       [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[PTRUE]], ptr [[TMP4]])
-; CHECK-NEXT:       [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
-; CHECK-NEXT:       [[EXT3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP5]], i64 0)
-; CHECK-NEXT:       [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
-; CHECK-NEXT:       [[EXT4:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP6]], i64 0)
-; CHECK-NEXT:       [[TMP7:%.*]] = shufflevector <4 x i64> [[EXT1]], <4 x i64> [[EXT3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       [[TMP8:%.*]] = shufflevector <4 x i64> [[EXT2]], <4 x i64> [[EXT4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @load_factor2_wide(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[PTR]], i32 8
+; CHECK-NEXT:    [[LDN1:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN1]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP7]], i64 0)
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN1]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP9]], i64 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <16 x i64>, ptr %ptr, align 4
   %v0 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %v1 = shufflevector <16 x i64> %interleaved.vec, <16 x i64> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -261,20 +296,22 @@ define void @load_factor2_wide(ptr %ptr) #0 {
 }
 
 define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
-; CHECK-LABEL:    @store_factor2_wide(
-; CHECK-NEXT:       [[PTRUE:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i64> %v0, <8 x i64> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:       [[INS1:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
-; CHECK-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i64> %v0, <8 x i64> %v1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:       [[INS2:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP3]], i64 0)
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[INS1]], <vscale x 2 x i64> [[INS2]], <vscale x 2 x i1> [[PTRUE]], ptr %ptr)
-; CHECK-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i64> %v0, <8 x i64> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:       [[INS3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
-; CHECK-NEXT:       [[TMP5:%.*]] = shufflevector <8 x i64> %v0, <8 x i64> %v1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:       [[INS4:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP5]], i64 0)
-; CHECK-NEXT:       [[TMP6:%.*]] = getelementptr i64, ptr %ptr, i32 8
-; CHECK-NEXT:       call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[INS3]], <vscale x 2 x i64> [[INS4]], <vscale x 2 x i1> [[PTRUE]], ptr [[TMP6]])
-; CHECK-NEXT:       ret void
+; CHECK-LABEL: define void @store_factor2_wide(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x i64> [[V0:%.*]], <8 x i64> [[V1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i64> [[V0]], <8 x i64> [[V1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP8]], i64 0)
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[PTR]], i32 8
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i1> [[TMP1]], ptr [[TMP10]])
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = shufflevector <8 x i64> %v0, <8 x i64> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   store <16 x i64> %interleaved.vec, ptr %ptr, align 4
   ret void
@@ -282,9 +319,27 @@ define void @store_factor2_wide(ptr %ptr, <8 x i64> %v0, <8 x i64> %v1) #0 {
 
 ; Check that neon is used for illegal multiples of 128-bit types
 define void @load_384bit(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_384bit(
-; CHECK: llvm.aarch64.neon.ld2
-; CHECK-NOT: llvm.aarch64.sve.ld2
+; CHECK-LABEL: define void @load_384bit(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[PTR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[PTR]], i32 4
+; CHECK-NEXT:    [[LDN1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[TMP3]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN1]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[TMP3]], i32 4
+; CHECK-NEXT:    [[LDN2:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN2]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN2]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP10]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP13]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <12 x i64>, ptr %ptr, align 4
   %v0 = shufflevector <12 x i64> %interleaved.vec, <12 x i64> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10>
   %v1 = shufflevector <12 x i64> %interleaved.vec, <12 x i64> poison, <6 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11>
@@ -293,9 +348,13 @@ define void @load_384bit(ptr %ptr) #0 {
 
 ; Check that neon is used for 128-bit vectors
 define void @load_128bit(ptr %ptr) #0 {
-; CHECK-LABEL:    @load_128bit(
-; CHECK: llvm.aarch64.neon.ld2
-; CHECK-NOT: llvm.aarch64.sve.ld2
+; CHECK-LABEL: define void @load_128bit(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr [[PTR]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <4 x i64>, ptr %ptr, align 4
   %v0 = shufflevector <4 x i64> %interleaved.vec, <4 x i64> poison, <2 x i32> <i32 0, i32 2>
   %v1 = shufflevector <4 x i64> %interleaved.vec, <4 x i64> poison, <2 x i32> <i32 1, i32 3>
@@ -304,8 +363,16 @@ define void @load_128bit(ptr %ptr) #0 {
 
 ; Check that correct ptrues are generated for min != max case
 define void @load_min_not_max(ptr %ptr) #1 {
-; CHECK-LABEL:    @load_min_not_max(
-; CHECK: call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-LABEL: define void @load_min_not_max(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <8 x i64>, ptr %ptr, align 4
   %v0 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %v1 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -313,8 +380,16 @@ define void @load_min_not_max(ptr %ptr) #1 {
 }
 
 define void @store_min_not_max(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
-; CHECK-LABEL:    @store_min_not_max(
-; CHECK: call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-LABEL: define void @store_min_not_max(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   store <8 x i64> %interleaved.vec, ptr %ptr, align 4
   ret void
@@ -322,8 +397,16 @@ define void @store_min_not_max(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #1 {
 
 ; Check that correct ptrues are generated for min > type case
 define void @load_min_ge_type(ptr %ptr) #2 {
-; CHECK-LABEL:    @load_min_ge_type(
-; CHECK: call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-LABEL: define void @load_min_ge_type(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = load <8 x i64>, ptr %ptr, align 4
   %v0 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   %v1 = shufflevector <8 x i64> %interleaved.vec, <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -331,25 +414,34 @@ define void @load_min_ge_type(ptr %ptr) #2 {
 }
 
 define void @store_min_ge_type(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1) #2 {
-; CHECK-LABEL:    @store_min_ge_type(
-; CHECK: call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-LABEL: define void @store_min_ge_type(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 4)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[V0]], <4 x i64> [[V1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP4]], i64 0)
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    ret void
+;
   %interleaved.vec = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   store <8 x i64> %interleaved.vec, ptr %ptr, align 4
   ret void
 }
 
 define void @load_double_factor4(ptr %ptr) #0 {
-; CHECK-LABEL: @load_double_factor4(
+; CHECK-LABEL: define void @load_double_factor4(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
-; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4.sret.nxv2f64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR:%.*]])
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 3
-; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP3]], i64 0)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP5]], i64 0)
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP7]], i64 0)
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP9]], i64 0)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4.sret.nxv2f64(<vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP6]], i64 0)
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[TMP8]], i64 0)
 ; CHECK-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x double>, ptr %ptr, align 4
@@ -361,15 +453,16 @@ define void @load_double_factor4(ptr %ptr) #0 {
 }
 
 define void @load_float_factor3(ptr %ptr) #0 {
-; CHECK-LABEL: @load_float_factor3(
+; CHECK-LABEL: define void @load_float_factor3(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
-; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3.sret.nxv4f32(<vscale x 4 x i1> [[TMP1]], ptr [[PTR:%.*]])
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP3]], i64 0)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP5]], i64 0)
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP7]], i64 0)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3.sret.nxv4f32(<vscale x 4 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> [[TMP6]], i64 0)
 ; CHECK-NEXT:    ret void
 ;
   %interleaved.vec = load <24 x float>, ptr %ptr, align 4
@@ -380,13 +473,14 @@ define void @load_float_factor3(ptr %ptr) #0 {
 }
 
 define void @load_half_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: @load_half_factor2(
+; CHECK-LABEL: define void @load_half_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2.sret.nxv8f16(<vscale x 8 x i1> [[TMP1]], ptr [[PTR:%.*]])
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[LDN]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x half> @llvm.vector.extract.v16f16.nxv8f16(<vscale x 8 x half> [[TMP3]], i64 0)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[LDN]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x half> @llvm.vector.extract.v16f16.nxv8f16(<vscale x 8 x half> [[TMP5]], i64 0)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2.sret.nxv8f16(<vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x half> @llvm.vector.extract.v16f16.nxv8f16(<vscale x 8 x half> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x half> @llvm.vector.extract.v16f16.nxv8f16(<vscale x 8 x half> [[TMP4]], i64 0)
 ; CHECK-NEXT:    ret void
 ;
   %interleaved.vec = load <32 x half>, ptr %ptr, align 4
@@ -396,13 +490,14 @@ define void @load_half_factor2(ptr %ptr) #0 {
 }
 
 define void @load_bfloat_factor2(ptr %ptr) #0 {
-; CHECK-LABEL: @load_bfloat_factor2(
+; CHECK-LABEL: define void @load_bfloat_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2.sret.nxv8bf16(<vscale x 8 x i1> [[TMP1]], ptr [[PTR:%.*]])
-; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[LDN]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[TMP3]], i64 0)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[LDN]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[TMP5]], i64 0)
+; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2.sret.nxv8bf16(<vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[LDN]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } [[LDN]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[TMP4]], i64 0)
 ; CHECK-NEXT:    ret void
 ;
   %interleaved.vec = load <32 x bfloat>, ptr %ptr, align 4
@@ -412,9 +507,10 @@ define void @load_bfloat_factor2(ptr %ptr) #0 {
 }
 
 define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) #0 {
-; CHECK-LABEL: @store_double_factor4(
-; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x double> [[V2:%.*]], <4 x double> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define void @store_double_factor4(
+; CHECK-SAME: ptr [[PTR:%.*]], <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x double> [[V2:%.*]], <4 x double> [[V3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x double> [[V0]], <4 x double> [[V1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x double> [[V2]], <4 x double> [[V3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP2]], i64 0)
@@ -424,7 +520,7 @@ define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1,
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP8]], i64 0)
-; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP7]], <vscale x 2 x double> [[TMP9]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR:%.*]])
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x double> [[TMP7]], <vscale x 2 x double> [[TMP9]], <vscale x 2 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
   %s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -435,9 +531,10 @@ define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1,
 }
 
 define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8 x float> %v2) #0 {
-; CHECK-LABEL: @store_float_factor3(
-; CHECK-NEXT:    [[S0:%.*]] = shufflevector <8 x float> [[V0:%.*]], <8 x float> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x float> [[V2:%.*]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-LABEL: define void @store_float_factor3(
+; CHECK-SAME: ptr [[PTR:%.*]], <8 x float> [[V0:%.*]], <8 x float> [[V1:%.*]], <8 x float> [[V2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <8 x float> [[V0]], <8 x float> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x float> [[V2]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[TMP2]], i64 0)
@@ -445,7 +542,7 @@ define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[TMP4]], i64 0)
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[TMP6]], i64 0)
-; CHECK-NEXT:    call void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP7]], <vscale x 4 x i1> [[TMP1]], ptr [[PTR:%.*]])
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x float> [[TMP7]], <vscale x 4 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
   %s0 = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -459,13 +556,14 @@ define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8
 }
 
 define void @store_half_factor2(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
-; CHECK-LABEL: @store_half_factor2(
+; CHECK-LABEL: define void @store_half_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]], <16 x half> [[V0:%.*]], <16 x half> [[V1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x half> [[V0:%.*]], <16 x half> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x half> [[V0]], <16 x half> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> undef, <16 x half> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x half> [[V0]], <16 x half> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v16f16(<vscale x 8 x half> undef, <16 x half> [[TMP4]], i64 0)
-; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR:%.*]])
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <16 x half> %v0, <16 x half> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
@@ -476,13 +574,14 @@ define void @store_half_factor2(ptr %ptr, <16 x half> %v0, <16 x half> %v1) #0 {
 
 
 define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1) #0 {
-; CHECK-LABEL: @store_bfloat_factor2(
+; CHECK-LABEL: define void @store_bfloat_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]], <16 x bfloat> [[V0:%.*]], <16 x bfloat> [[V1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x bfloat> [[V0:%.*]], <16 x bfloat> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x bfloat> [[V0]], <16 x bfloat> [[V1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> undef, <16 x bfloat> [[TMP2]], i64 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x bfloat> [[V0]], <16 x bfloat> [[V1]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> undef, <16 x bfloat> [[TMP4]], i64 0)
-; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR:%.*]])
+; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x i1> [[TMP1]], ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
   %interleaved.vec = shufflevector <16 x bfloat> %v0, <16 x bfloat> %v1, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23,
@@ -492,8 +591,9 @@ define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1
 }
 
 ; Ensure vscale_range property does not affect scalable vector types.
-define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_nxptr_factor2(ptr %ptr) #2 {
-; CHECK-LABEL: define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_nxptr_factor2(
+define void @deinterleave_nxptr_factor2(ptr %ptr) #2 {
+; CHECK-LABEL: define void @deinterleave_nxptr_factor2(
+; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[PTR]], i64 0
 ; CHECK-NEXT:    [[LDN1:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), ptr [[TMP1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN1]], 0
@@ -506,13 +606,13 @@ define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_nxptr_fact
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP7]], i64 2)
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN2]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP5]], <vscale x 2 x double> [[TMP9]], i64 2)
-; CHECK-NEXT:    [[TMP11:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } poison, <vscale x 4 x double> [[TMP8]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP11]], <vscale x 4 x double> [[TMP10]], 1
-; CHECK-NEXT:    ret { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP12]]
+; CHECK-NEXT:    ret void
 ;
   %wide.vec = load <vscale x 8 x double>, ptr %ptr, align 8
   %ldN = tail call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %wide.vec)
-  ret { <vscale x 4 x double>, <vscale x 4 x double> } %ldN
+  %extract1 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %ldN, 0
+  %extract2 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %ldN, 1
+  ret void
 }
 
 attributes #0 = { vscale_range(2,2) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 9c9547c8b6322d7..9f70a891efe76b6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -52,8 +52,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
-; CHECK-NEXT:  LV: Loop does not require scalar epilogue
-; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
 ; CHECK-NEXT:  Live-in vp<%0> = VF * UF
 ; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
@@ -257,8 +255,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
-; CHECK-NEXT:  LV: Loop does not require scalar epilogue
-; CHECK-NEXT:  LV: Loop does not require scalar epilogue
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
 ; CHECK-NEXT:  Live-in vp<%0> = VF * UF
 ; CHECK-NEXT:  Live-in vp<%1> = vector-trip-count
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
index 9c3e1fbec663baf..c97d8da58be4418 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
@@ -86,16 +86,17 @@ return:
 define float @test_merge_anyof_v4sf(<4 x float> %t) {
 ; CHECK-LABEL: @test_merge_anyof_v4sf(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[T_FR:%.*]] = freeze <4 x float> [[T:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = fcmp olt <4 x float> [[T_FR]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt <4 x float> [[T_FR]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i1> [[TMP1]], [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
-; CHECK-NEXT:    [[OP_RDX_NOT:%.*]] = icmp eq i4 [[TMP3]], 0
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[T_FR]], [[SHIFT]]
-; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP4]], i64 0
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[OP_RDX_NOT]], float [[ADD]], float 0.000000e+00
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x float> [[T:%.*]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt <8 x float> [[TMP0]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp olt <8 x float> [[TMP0]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <8 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i8 [[TMP5]], 0
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[SHIFT]], [[T]]
+; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP6]], i64 0
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[DOTNOT]], float [[ADD]], float 0.000000e+00
 ; CHECK-NEXT:    ret float [[RETVAL_0]]
 ;
 entry:
@@ -400,16 +401,18 @@ return:
 define float @test_merge_anyof_v4si(<4 x i32> %t) {
 ; CHECK-LABEL: @test_merge_anyof_v4si(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[T_FR:%.*]] = freeze <4 x i32> [[T:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i32> [[T_FR]], <i32 -256, i32 -256, i32 -256, i32 -256>
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i32> [[TMP0]], <i32 -255, i32 -255, i32 -255, i32 -255>
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i1> [[TMP1]] to i4
-; CHECK-NEXT:    [[OP_RDX_NOT:%.*]] = icmp eq i4 [[TMP2]], 0
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[T_FR]], [[SHIFT]]
-; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[T:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <8 x i32> [[TMP0]], <i32 255, i32 255, i32 255, i32 255, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[TMP0]], <i32 255, i32 255, i32 255, i32 255, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = freeze <8 x i1> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i8 [[TMP5]], 0
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[SHIFT]], [[T]]
+; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP6]], i64 0
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[ADD]] to float
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[OP_RDX_NOT]], float [[CONV]], float 0.000000e+00
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[DOTNOT]], float [[CONV]], float 0.000000e+00
 ; CHECK-NEXT:    ret float [[RETVAL_0]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll
index c25e07cfd451fce..865d81786671678 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-logical.ll
@@ -388,28 +388,16 @@ define i1 @logical_and_icmp_clamp_partial(<4 x i32> %x) {
 }
 
 define i1 @logical_and_icmp_clamp_pred_diff(<4 x i32> %x) {
-; SSE-LABEL: @logical_and_icmp_clamp_pred_diff(
-; SSE-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[X:%.*]], <i32 42, i32 42, i32 42, i32 42>
-; SSE-NEXT:    [[TMP2:%.*]] = icmp ult <4 x i32> [[X]], <i32 42, i32 42, i32 42, i32 42>
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = icmp sgt <4 x i32> [[X]], <i32 17, i32 17, i32 17, i32 17>
-; SSE-NEXT:    [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
-; SSE-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP5]])
-; SSE-NEXT:    [[TMP7:%.*]] = freeze <4 x i1> [[TMP3]]
-; SSE-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP7]])
-; SSE-NEXT:    [[OP_RDX:%.*]] = select i1 [[TMP6]], i1 [[TMP8]], i1 false
-; SSE-NEXT:    ret i1 [[OP_RDX]]
-;
-; AVX-LABEL: @logical_and_icmp_clamp_pred_diff(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 42, i32 42, i32 42, i32 poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 3>
-; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> <i32 17, i32 17, i32 17, i32 17, i32 poison, i32 poison, i32 poison, i32 42>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 15>
-; AVX-NEXT:    [[TMP4:%.*]] = icmp sgt <8 x i32> [[TMP2]], [[TMP3]]
-; AVX-NEXT:    [[TMP5:%.*]] = icmp ult <8 x i32> [[TMP2]], [[TMP3]]
-; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
-; AVX-NEXT:    [[TMP7:%.*]] = freeze <8 x i1> [[TMP6]]
-; AVX-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP7]])
-; AVX-NEXT:    ret i1 [[TMP8]]
+; CHECK-LABEL: @logical_and_icmp_clamp_pred_diff(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 42, i32 42, i32 42, i32 poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> <i32 17, i32 17, i32 17, i32 17, i32 poison, i32 poison, i32 poison, i32 42>, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = freeze <8 x i1> [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP7]])
+; CHECK-NEXT:    ret i1 [[TMP8]]
 ;
   %x0 = extractelement <4 x i32> %x, i32 0
   %x1 = extractelement <4 x i32> %x, i32 1
diff --git a/llvm/test/tools/obj2yaml/Minidump/basic.yaml b/llvm/test/tools/obj2yaml/Minidump/basic.yaml
index dfabc132d8e71c5..3df4689d6761ea4 100644
--- a/llvm/test/tools/obj2yaml/Minidump/basic.yaml
+++ b/llvm/test/tools/obj2yaml/Minidump/basic.yaml
@@ -67,9 +67,16 @@ Streams:
       Parameter 1: 0x24
     Thread Context:  '8182838485868788'
   - Type:            MemoryList
-    Memory Ranges:   
+    Memory Ranges:
       - Start of Memory Range: 0x7C7D7E7F80818283
         Content:               '8485868788'
+  - Type:            Memory64List
+    Memory Ranges:
+      - Start of Memory Range: 0x7FFFFFCF08180283
+        Content:               '68656c6c6f776f726c64'
+      - Start of Memory Range: 0x7FFAFFCF08180283
+        Data Size:              8
+        Content:               '8008'
   - Type:            MemoryInfoList
     Memory Ranges:
       - Base Address:    0x0000000000000000
@@ -156,9 +163,15 @@ Streams:
 # CHECK-NEXT:       Parameter 1: 0x24
 # CHECK-NEXT:     Thread Context:  '8182838485868788'
 # CHECK-NEXT:   - Type:            MemoryList
-# CHECK-NEXT:     Memory Ranges:   
+# CHECK-NEXT:     Memory Ranges:
 # CHECK-NEXT:       - Start of Memory Range: 0x7C7D7E7F80818283
 # CHECK-NEXT:         Content:               '8485868788'
+# CHECK-NEXT:  - Type:            Memory64List
+# CHECK-NEXT:    Memory Ranges:
+# CHECK-NEXT:      - Start of Memory Range: 0x7FFFFFCF08180283
+# CHECK-NEXT:        Content:         68656C6C6F776F726C64
+# CHECK-NEXT:      - Start of Memory Range: 0x7FFAFFCF08180283
+# CHECK-NEXT:        Content:         '8008000000000000'
 # CHECK-NEXT:   - Type:            MemoryInfoList
 # CHECK-NEXT:     Memory Ranges:
 # CHECK-NEXT:       - Base Address:       0x0
diff --git a/llvm/unittests/ObjectYAML/MinidumpYAMLTest.cpp b/llvm/unittests/ObjectYAML/MinidumpYAMLTest.cpp
index 0297f62d1069982..a8b8da925d21d9a 100644
--- a/llvm/unittests/ObjectYAML/MinidumpYAMLTest.cpp
+++ b/llvm/unittests/ObjectYAML/MinidumpYAMLTest.cpp
@@ -336,3 +336,66 @@ TEST(MinidumpYAML, ExceptionStream_ExtraParameter) {
                                0xab, 0xad, 0xca, 0xfe}),
             *ExpectedContext);
 }
+
+TEST(MinidumpYAML, MemoryRegion_64bit) {
+  SmallString<0> Storage;
+  auto ExpectedFile = toBinary(Storage, R"(
+--- !minidump
+Streams:
+  - Type:            Memory64List
+    Memory Ranges:
+      - Start of Memory Range: 0x7FFFFFCF0818283
+        Content:               '68656c6c6f'
+      - Start of Memory Range: 0x7FFFFFFF0818283
+        Content:               '776f726c64'
+        )");
+
+  ASSERT_THAT_EXPECTED(ExpectedFile, Succeeded());
+  object::MinidumpFile &File = **ExpectedFile;
+
+  ASSERT_THAT(File.streams().size(), 1u);
+
+  Error Err = Error::success();
+  iterator_range<object::MinidumpFile::FallibleMemory64Iterator> MemoryList =
+      File.getMemory64List(Err);
+
+  ASSERT_THAT_ERROR(std::move(Err), Succeeded());
+  auto Iterator = MemoryList.begin();
+
+  auto DescOnePair = *Iterator;
+  const minidump::MemoryDescriptor_64 &DescOne = DescOnePair.first;
+  ASSERT_THAT(DescOne.StartOfMemoryRange, 0x7FFFFFCF0818283u);
+  ASSERT_THAT(DescOne.DataSize, 5u);
+  ++Iterator;
+  ASSERT_THAT_ERROR(std::move(Err), Succeeded());
+
+  auto DescTwoPair = *Iterator;
+  const minidump::MemoryDescriptor_64 &DescTwo = DescTwoPair.first;
+  ASSERT_THAT(DescTwo.StartOfMemoryRange, 0x7FFFFFFF0818283u);
+  ASSERT_THAT(DescTwo.DataSize, 5u);
+  ++Iterator;
+  ASSERT_THAT_ERROR(std::move(Err), Succeeded());
+
+  const std::optional<ArrayRef<uint8_t>> ExpectedContent =
+      File.getRawStream(StreamType::Memory64List);
+  ASSERT_TRUE(ExpectedContent);
+  const size_t ExpectedStreamSize =
+      sizeof(Memory64ListHeader) + (sizeof(MemoryDescriptor_64) * 2);
+  ASSERT_THAT(ExpectedContent->size(), ExpectedStreamSize);
+
+  Expected<minidump::Memory64ListHeader> ExpectedHeader =
+      File.getMemoryList64Header();
+  ASSERT_THAT_EXPECTED(ExpectedHeader, Succeeded());
+  ASSERT_THAT(ExpectedHeader->BaseRVA, 92u);
+
+  Expected<ArrayRef<uint8_t>> DescOneExpectedContentSlice = DescOnePair.second;
+  ASSERT_THAT_EXPECTED(DescOneExpectedContentSlice, Succeeded());
+  ASSERT_THAT(DescOneExpectedContentSlice->size(), 5u);
+  ASSERT_THAT(*DescOneExpectedContentSlice, arrayRefFromStringRef("hello"));
+
+  Expected<ArrayRef<uint8_t>> DescTwoExpectedContentSlice = DescTwoPair.second;
+  ASSERT_THAT_EXPECTED(DescTwoExpectedContentSlice, Succeeded());
+  ASSERT_THAT(*DescTwoExpectedContentSlice, arrayRefFromStringRef("world"));
+
+  ASSERT_EQ(Iterator, MemoryList.end());
+}
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index f98a60b49ecab36..1cd1ca6a418c612 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -631,6 +631,50 @@ define void @foo(i1 %c0, i8 %v0, i8 %v1, i1 %c1) {
   }
 }
 
+TEST_F(SandboxIRTest, ExtractElementInst) {
+  parseIR(C, R"IR(
+define void @foo(<2 x i8> %vec, i32 %idx) {
+  %ins0 = extractelement <2 x i8> %vec, i32 %idx
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *ArgVec = F.getArg(0);
+  auto *ArgIdx = F.getArg(1);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *EI = cast<sandboxir::ExtractElementInst>(&*It++);
+  auto *Ret = &*It++;
+
+  EXPECT_EQ(EI->getOpcode(), sandboxir::Instruction::Opcode::ExtractElement);
+  EXPECT_EQ(EI->getOperand(0), ArgVec);
+  EXPECT_EQ(EI->getOperand(1), ArgIdx);
+  EXPECT_EQ(EI->getVectorOperand(), ArgVec);
+  EXPECT_EQ(EI->getIndexOperand(), ArgIdx);
+  EXPECT_EQ(EI->getVectorOperandType(), ArgVec->getType());
+
+  auto *NewI1 =
+      cast<sandboxir::ExtractElementInst>(sandboxir::ExtractElementInst::create(
+          ArgVec, ArgIdx, Ret, Ctx, "NewExtrBeforeRet"));
+  EXPECT_EQ(NewI1->getOperand(0), ArgVec);
+  EXPECT_EQ(NewI1->getOperand(1), ArgIdx);
+  EXPECT_EQ(NewI1->getNextNode(), Ret);
+
+  auto *NewI2 =
+      cast<sandboxir::ExtractElementInst>(sandboxir::ExtractElementInst::create(
+          ArgVec, ArgIdx, BB, Ctx, "NewExtrAtEndOfBB"));
+  EXPECT_EQ(NewI2->getPrevNode(), Ret);
+
+  auto *LLVMArgVec = LLVMF.getArg(0);
+  auto *LLVMArgIdx = LLVMF.getArg(1);
+  EXPECT_EQ(sandboxir::ExtractElementInst::isValidOperands(ArgVec, ArgIdx),
+            llvm::ExtractElementInst::isValidOperands(LLVMArgVec, LLVMArgIdx));
+  EXPECT_EQ(sandboxir::ExtractElementInst::isValidOperands(ArgIdx, ArgVec),
+            llvm::ExtractElementInst::isValidOperands(LLVMArgIdx, LLVMArgVec));
+}
+
 TEST_F(SandboxIRTest, InsertElementInst) {
   parseIR(C, R"IR(
 define void @foo(i8 %v0, i8 %v1, <2 x i8> %vec) {
diff --git a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index ac3a3d863bb7802..1dc4c3f14b930aa 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -40,6 +40,7 @@ static_library("Analysis") {
     "CycleAnalysis.cpp",
     "DDG.cpp",
     "DDGPrinter.cpp",
+    "DXILMetadataAnalysis.cpp",
     "DXILResource.cpp",
     "Delinearization.cpp",
     "DemandedBits.cpp",
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index b5bb2f42f2961c0..7bde9e490e4f4ec 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -1300,7 +1300,9 @@ def ConvertVectorToSCF : Pass<"convert-vector-to-scf"> {
     Option<"targetRank", "target-rank", "unsigned", /*default=*/"1",
            "Target vector rank to which transfer ops should be lowered">,
     Option<"lowerTensors", "lower-tensors", "bool", /*default=*/"false",
-           "Lower transfer ops that operate on tensors">
+           "Lower transfer ops that operate on tensors">,
+    Option<"lowerScalable", "lower-scalable", "bool", /*default=*/"false",
+           "Add scalable vector specific lowerings (that introduce loops)">
   ];
 }
 
diff --git a/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h b/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h
index 1c834b6c690830f..e0ef67c39a1013b 100644
--- a/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h
+++ b/mlir/include/mlir/Conversion/VectorToSCF/VectorToSCF.h
@@ -69,6 +69,14 @@ struct VectorTransferToSCFOptions {
     unroll = u;
     return *this;
   }
+  /// Enable scalable vector specific lowerings (which introduce loops). These
+  /// work alongside fullUnroll (which unrolls until the first scalable
+  /// dimension).
+  bool lowerScalable = false;
+  VectorTransferToSCFOptions enableLowerScalable(bool enable = true) {
+    lowerScalable = enable;
+    return *this;
+  }
 };
 
 /// Collect a set of patterns to convert from the Vector dialect to SCF + func.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
index 7dc5d0522910e3e..edcc34461f2f264 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
@@ -32,6 +32,7 @@ def LLVM_Dialect : Dialect {
     static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; }
     static StringRef getAliasScopesAttrName() { return "alias_scopes"; }
     static StringRef getAccessGroupsAttrName() { return "access_groups"; }
+    static StringRef getIdentAttrName() { return "llvm.ident"; }
 
     /// Names of llvm parameter attributes.
     static StringRef getAlignAttrName() { return "llvm.align"; }
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 68f92e6952694b6..d63fdd88f791048 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -128,6 +128,7 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove, RecipeInterface]>
 
 def ParallelOp : OpenMP_Op<"parallel", traits = [
     AttrSizedOperandSegments, AutomaticAllocationScope,
+    DeclareOpInterfaceMethods<ComposableOpInterface>,
     DeclareOpInterfaceMethods<LoopWrapperInterface>,
     DeclareOpInterfaceMethods<OutlineableOpenMPOpInterface>,
     RecursiveMemoryEffects
@@ -356,7 +357,9 @@ def LoopNestOp : OpenMP_Op<"loop_nest", traits = [
 //===----------------------------------------------------------------------===//
 
 def WsloopOp : OpenMP_Op<"wsloop", traits = [
-    AttrSizedOperandSegments, DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    AttrSizedOperandSegments,
+    DeclareOpInterfaceMethods<ComposableOpInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>,
     RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AllocateClauseSkip<assemblyFormat = true>,
@@ -432,7 +435,9 @@ def WsloopOp : OpenMP_Op<"wsloop", traits = [
 //===----------------------------------------------------------------------===//
 
 def SimdOp : OpenMP_Op<"simd", traits = [
-    AttrSizedOperandSegments, DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    AttrSizedOperandSegments,
+    DeclareOpInterfaceMethods<ComposableOpInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>,
     RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AlignedClause, OpenMP_IfClause, OpenMP_LinearClause,
@@ -499,7 +504,9 @@ def YieldOp : OpenMP_Op<"yield",
 // Distribute construct [2.9.4.1]
 //===----------------------------------------------------------------------===//
 def DistributeOp : OpenMP_Op<"distribute", traits = [
-    AttrSizedOperandSegments, DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    AttrSizedOperandSegments,
+    DeclareOpInterfaceMethods<ComposableOpInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>,
     RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AllocateClause, OpenMP_DistScheduleClause, OpenMP_OrderClause,
@@ -587,8 +594,9 @@ def TaskOp : OpenMP_Op<"task", traits = [
 
 def TaskloopOp : OpenMP_Op<"taskloop", traits = [
     AttrSizedOperandSegments, AutomaticAllocationScope,
-    DeclareOpInterfaceMethods<LoopWrapperInterface>, RecursiveMemoryEffects,
-    SingleBlock
+    DeclareOpInterfaceMethods<ComposableOpInterface>,
+    DeclareOpInterfaceMethods<LoopWrapperInterface>,
+    RecursiveMemoryEffects, SingleBlock
   ], clauses = [
     OpenMP_AllocateClause, OpenMP_FinalClause, OpenMP_GrainsizeClause,
     OpenMP_IfClause, OpenMP_InReductionClauseSkip<extraClassDeclaration = true>,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 2d1de37239c82a1..78637f0ab8c2da6 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -142,6 +142,42 @@ def LoopWrapperInterface : OpInterface<"LoopWrapperInterface"> {
   ];
 }
 
+def ComposableOpInterface : OpInterface<"ComposableOpInterface"> {
+  let description = [{
+    OpenMP operations that can represent a single leaf of a composite OpenMP
+    construct.
+  }];
+
+  let cppNamespace = "::mlir::omp";
+
+  let methods = [
+    InterfaceMethod<
+      /*description=*/[{
+        Check whether the operation is representing a leaf of a composite OpenMP
+        construct.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isComposite",
+      (ins ), [{}], [{
+        return $_op->hasAttr("omp.composite");
+      }]
+    >,
+    InterfaceMethod<
+      /*description=*/[{
+        Mark the operation as part of an OpenMP composite construct.
+      }],
+      /*retTy=*/"void",
+      /*methodName=*/"setComposite",
+      (ins "bool":$val), [{}], [{
+        if (val)
+          $_op->setDiscardableAttr("omp.composite", mlir::UnitAttr::get($_op->getContext()));
+        else
+          $_op->removeDiscardableAttr("omp.composite");
+      }]
+    >
+  ];
+}
+
 def DeclareTargetInterface : OpInterface<"DeclareTargetInterface"> {
   let description = [{
     OpenMP operations that support declare target have this interface.
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
index 5364e2ad4732540..11f62c283d6a96e 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
@@ -196,6 +196,9 @@ class ModuleImport {
   /// LLVM dialect operation.
   LogicalResult convertLinkerOptionsMetadata();
 
+  /// Converts !llvm.ident metadata to the llvm.ident LLVM ModuleOp attribute.
+  LogicalResult convertIdentMetadata();
+
   /// Converts all LLVM metadata nodes that translate to attributes such as
   /// alias analysis or access group metadata, and builds a map from the
   /// metadata nodes to the converted attributes.
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 85fdfed3bdbeb2b..3c85338bc642f69 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -329,6 +329,9 @@ class ModuleTranslation {
   /// metadata nodes for them.
   LogicalResult createTBAAMetadata();
 
+  /// Process the ident LLVM Metadata, if it exists.
+  LogicalResult createIdentMetadata();
+
   /// Translates dialect attributes attached to the given operation.
   LogicalResult
   convertDialectAttributes(Operation *op,
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 0b29c93e2d8909e..5b3c2fb15e70263 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -414,6 +414,7 @@ void populateMathToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
            ExpM1OpPattern<spirv::GLExpOp>, PowFOpPattern, RoundOpPattern,
            CheckedElementwiseOpPattern<math::AbsFOp, spirv::GLFAbsOp>,
            CheckedElementwiseOpPattern<math::AbsIOp, spirv::GLSAbsOp>,
+           CheckedElementwiseOpPattern<math::AtanOp, spirv::GLAtanOp>,
            CheckedElementwiseOpPattern<math::CeilOp, spirv::GLCeilOp>,
            CheckedElementwiseOpPattern<math::CosOp, spirv::GLCosOp>,
            CheckedElementwiseOpPattern<math::ExpOp, spirv::GLExpOp>,
@@ -431,6 +432,8 @@ void populateMathToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
   patterns.add<Log1pOpPattern<spirv::CLLogOp>, ExpM1OpPattern<spirv::CLExpOp>,
                CheckedElementwiseOpPattern<math::AbsFOp, spirv::CLFAbsOp>,
                CheckedElementwiseOpPattern<math::AbsIOp, spirv::CLSAbsOp>,
+               CheckedElementwiseOpPattern<math::AtanOp, spirv::CLAtanOp>,
+               CheckedElementwiseOpPattern<math::Atan2Op, spirv::CLAtan2Op>,
                CheckedElementwiseOpPattern<math::CeilOp, spirv::CLCeilOp>,
                CheckedElementwiseOpPattern<math::CosOp, spirv::CLCosOp>,
                CheckedElementwiseOpPattern<math::ErfOp, spirv::CLErfOp>,
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 16f1db44acc3514..2372ab5b82a772e 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -15,6 +15,7 @@
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Transforms/Transforms.h"
 #include "mlir/IR/Builders.h"
@@ -370,9 +371,18 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   auto comparison = rewriter.create<arith::CmpIOp>(
       loc, arith::CmpIPredicate::slt, iv, upperBound);
 
-  rewriter.create<cf::CondBranchOp>(loc, comparison, firstBodyBlock,
-                                    ArrayRef<Value>(), endBlock,
-                                    ArrayRef<Value>());
+  auto condBranchOp = rewriter.create<cf::CondBranchOp>(
+      loc, comparison, firstBodyBlock, ArrayRef<Value>(), endBlock,
+      ArrayRef<Value>());
+
+  // Let the CondBranchOp carry the LLVM attributes from the ForOp, such as the
+  // llvm.loop_annotation attribute.
+  SmallVector<NamedAttribute> llvmAttrs;
+  llvm::copy_if(forOp->getAttrs(), std::back_inserter(llvmAttrs),
+                [](auto attr) {
+                  return isa<LLVM::LLVMDialect>(attr.getValue().getDialect());
+                });
+  condBranchOp->setDiscardableAttrs(llvmAttrs);
   // The result of the loop operation is the values of the condition block
   // arguments except the induction variable on the last iteration.
   rewriter.replaceOp(forOp, conditionBlock->getArguments().drop_front());
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index 19f02297bfbb714..3a4dc806efe976a 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -24,6 +24,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
+#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Pass/Pass.h"
@@ -44,6 +45,18 @@ namespace {
 /// Attribute name used for labeling transfer ops during progressive lowering.
 static const char kPassLabel[] = "__vector_to_scf_lowering__";
 
+/// Return true if this transfer op operates on a source tensor.
+static bool isTensorOp(VectorTransferOpInterface xferOp) {
+  if (isa<RankedTensorType>(xferOp.getShapedType())) {
+    if (isa<vector::TransferWriteOp>(xferOp)) {
+      // TransferWriteOps on tensors have a result.
+      assert(xferOp->getNumResults() > 0);
+    }
+    return true;
+  }
+  return false;
+}
+
 /// Patterns that inherit from this struct have access to
 /// VectorTransferToSCFOptions.
 template <typename OpTy>
@@ -52,6 +65,15 @@ struct VectorToSCFPattern : public OpRewritePattern<OpTy> {
                               VectorTransferToSCFOptions opt)
       : OpRewritePattern<OpTy>(context), options(opt) {}
 
+  LogicalResult checkLowerTensors(VectorTransferOpInterface xferOp,
+                                  PatternRewriter &rewriter) const {
+    if (isTensorOp(xferOp) && !options.lowerTensors) {
+      return rewriter.notifyMatchFailure(
+          xferOp, "lowering tensor transfers is disabled");
+    }
+    return success();
+  }
+
   VectorTransferToSCFOptions options;
 };
 
@@ -257,19 +279,6 @@ static void maybeApplyPassLabel(OpBuilder &b, OpTy newXferOp,
     newXferOp->setAttr(kPassLabel, b.getUnitAttr());
 }
 
-/// Return true if this transfer op operates on a source tensor.
-template <typename OpTy>
-static bool isTensorOp(OpTy xferOp) {
-  if (isa<RankedTensorType>(xferOp.getShapedType())) {
-    if (xferOp.getOperationName() == TransferWriteOp::getOperationName()) {
-      // TransferWriteOps on tensors have a result.
-      assert(xferOp->getNumResults() > 0);
-    }
-    return true;
-  }
-  return false;
-}
-
 namespace lowering_n_d {
 
 /// Helper data structure for data and mask buffers.
@@ -987,6 +996,189 @@ struct TransferOpConversion : public VectorToSCFPattern<OpTy> {
   }
 };
 
+/// Retrieves the dimensions sizes of a mask. Currently supports CreateMaskOp
+/// and ConstantMaskOp.
+template <typename VscaleConstantBuilder>
+static FailureOr<SmallVector<OpFoldResult>>
+getMaskDimSizes(Value mask, VscaleConstantBuilder &createVscaleMultiple) {
+  if (!mask)
+    return SmallVector<OpFoldResult>{};
+  if (auto createMaskOp = mask.getDefiningOp<vector::CreateMaskOp>()) {
+    return llvm::map_to_vector(createMaskOp.getOperands(), [](Value dimSize) {
+      return OpFoldResult(dimSize);
+    });
+  }
+  if (auto constantMask = mask.getDefiningOp<vector::ConstantMaskOp>()) {
+    int dimIdx = 0;
+    VectorType maskType = constantMask.getVectorType();
+    auto indexType = IndexType::get(mask.getContext());
+    return llvm::map_to_vector(
+        constantMask.getMaskDimSizes(), [&](int64_t dimSize) {
+          // A scalable dim in a constant_mask means vscale x dimSize.
+          if (maskType.getScalableDims()[dimIdx++])
+            return OpFoldResult(createVscaleMultiple(dimSize));
+          return OpFoldResult(IntegerAttr::get(indexType, dimSize));
+        });
+  }
+  return failure();
+}
+
+/// Scalable vector lowering of transfer_write(transpose). This lowering only
+/// supports rank 2 (scalable) vectors, but can be used in conjunction with
+/// `UnrollTransferWriteConversion` to support n-D cases. The unroll conversion
+/// unrolls until the first scalable dimension.
+///
+/// Example:
+///
+/// BEFORE:
+/// ```mlir
+/// %transpose = vector.transpose %vec, [1, 0]
+///    : vector<4x[4]xf32> to vector<[4]x4xf32>
+/// vector.transfer_write %transpose, %dest[%i, %j] {in_bounds = [true, true]}
+///    : vector<[4]x4xf32>,  memref<?x?xf32>
+/// ```
+///
+/// AFTER:
+/// ```mlir
+/// %c1 = arith.constant 1 : index
+/// %c4 = arith.constant 4 : index
+/// %c0 = arith.constant 0 : index
+/// %0 = vector.extract %arg0[0] : vector<[4]xf32> from vector<4x[4]xf32>
+/// %1 = vector.extract %arg0[1] : vector<[4]xf32> from vector<4x[4]xf32>
+/// %2 = vector.extract %arg0[2] : vector<[4]xf32> from vector<4x[4]xf32>
+/// %3 = vector.extract %arg0[3] : vector<[4]xf32> from vector<4x[4]xf32>
+/// %vscale = vector.vscale
+/// %c4_vscale = arith.muli %vscale, %c4 : index
+/// scf.for %idx = %c0 to %c4_vscale step %c1 {
+///   %4 = vector.extract %0[%idx] : f32 from vector<[4]xf32>
+///   %5 = vector.extract %1[%idx] : f32 from vector<[4]xf32>
+///   %6 = vector.extract %2[%idx] : f32 from vector<[4]xf32>
+///   %7 = vector.extract %3[%idx] : f32 from vector<[4]xf32>
+///   %slice_i = affine.apply #map(%idx)[%i]
+///   %slice = vector.from_elements %4, %5, %6, %7 : vector<4xf32>
+///   vector.transfer_write %slice, %arg1[%slice_i, %j] {in_bounds = [true]}
+///     : vector<4xf32>, memref<?x?xf32>
+/// }
+/// ```
+struct ScalableTransposeTransferWriteConversion
+    : VectorToSCFPattern<vector::TransferWriteOp> {
+  using VectorToSCFPattern::VectorToSCFPattern;
+
+  LogicalResult matchAndRewrite(TransferWriteOp writeOp,
+                                PatternRewriter &rewriter) const override {
+    if (failed(checkLowerTensors(writeOp, rewriter)))
+      return failure();
+
+    VectorType vectorType = writeOp.getVectorType();
+
+    // Note: By comparing the scalable dims to an ArrayRef of length two this
+    // implicitly checks the rank (is also two).
+    ArrayRef<bool> scalableFlags = vectorType.getScalableDims();
+    if (scalableFlags != ArrayRef<bool>{true, false}) {
+      return rewriter.notifyMatchFailure(
+          writeOp, "expected vector of the form vector<[N]xMxty>");
+    }
+
+    auto permutationMap = writeOp.getPermutationMap();
+    if (!permutationMap.isIdentity()) {
+      return rewriter.notifyMatchFailure(
+          writeOp, "non-identity permutations are unsupported (lower first)");
+    }
+
+    // Note: This pattern is only lowering the leading dimension (to a loop),
+    // so we only check if the leading dimension is in bounds. The in-bounds
+    // attribute for the trailing dimension will be propagated.
+    if (!writeOp.isDimInBounds(0)) {
+      return rewriter.notifyMatchFailure(
+          writeOp, "out-of-bounds dims are unsupported (use masking)");
+    }
+
+    Value vector = writeOp.getVector();
+    auto transposeOp = vector.getDefiningOp<vector::TransposeOp>();
+    if (!transposeOp ||
+        transposeOp.getPermutation() != ArrayRef<int64_t>{1, 0}) {
+      return rewriter.notifyMatchFailure(writeOp, "source not transpose");
+    }
+
+    auto loc = writeOp.getLoc();
+    auto createVscaleMultiple =
+        vector::makeVscaleConstantBuilder(rewriter, loc);
+
+    auto maskDims = getMaskDimSizes(writeOp.getMask(), createVscaleMultiple);
+    if (failed(maskDims)) {
+      return rewriter.notifyMatchFailure(writeOp,
+                                         "failed to resolve mask dims");
+    }
+
+    int64_t fixedDimSize = vectorType.getDimSize(1);
+    auto fixedDimOffsets = llvm::seq(fixedDimSize);
+
+    // Extract all slices from the source of the transpose.
+    auto transposeSource = transposeOp.getVector();
+    SmallVector<Value> transposeSourceSlices =
+        llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
+          return rewriter.create<vector::ExtractOp>(loc, transposeSource, idx);
+        });
+
+    // Loop bounds and step.
+    auto lb = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto ub =
+        maskDims->empty()
+            ? Value(createVscaleMultiple(vectorType.getDimSize(0)))
+            : vector::getAsValues(rewriter, loc, maskDims->front()).front();
+    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+
+    // Generate a new mask for the slice.
+    VectorType sliceType = VectorType::Builder(vectorType).dropDim(0);
+    Value sliceMask = nullptr;
+    if (!maskDims->empty()) {
+      sliceMask = rewriter.create<vector::CreateMaskOp>(
+          loc, sliceType.clone(rewriter.getI1Type()),
+          ArrayRef<OpFoldResult>(*maskDims).drop_front());
+    }
+
+    Value initDest = isTensorOp(writeOp) ? writeOp.getSource() : Value{};
+    ValueRange initLoopArgs = initDest ? initDest : ValueRange{};
+    auto result = rewriter.create<scf::ForOp>(
+        loc, lb, ub, step, initLoopArgs,
+        [&](OpBuilder &b, Location loc, Value iv, ValueRange loopIterArgs) {
+          // Indices for the new transfer op.
+          SmallVector<Value, 8> xferIndices;
+          getXferIndices(b, writeOp, iv, xferIndices);
+
+          // Extract a transposed slice from the source vector.
+          SmallVector<Value> transposeElements =
+              llvm::map_to_vector(fixedDimOffsets, [&](int64_t idx) -> Value {
+                return b.create<vector::ExtractOp>(
+                    loc, transposeSourceSlices[idx], iv);
+              });
+          auto sliceVec = b.create<vector::FromElementsOp>(loc, sliceType,
+                                                           transposeElements);
+
+          // Create the transfer_write for the slice.
+          Value dest =
+              loopIterArgs.empty() ? writeOp.getSource() : loopIterArgs.front();
+          auto newWriteOp = b.create<vector::TransferWriteOp>(
+              loc, sliceVec, dest, xferIndices,
+              ArrayRef<bool>(writeOp.getInBoundsValues()).drop_front());
+          if (sliceMask)
+            newWriteOp.getMaskMutable().assign(sliceMask);
+
+          // Yield from the loop.
+          b.create<scf::YieldOp>(loc, loopIterArgs.empty()
+                                          ? ValueRange{}
+                                          : newWriteOp.getResult());
+        });
+
+    if (isTensorOp(writeOp))
+      rewriter.replaceOp(writeOp, result);
+    else
+      rewriter.eraseOp(writeOp);
+
+    return success();
+  }
+};
+
 } // namespace lowering_n_d
 
 namespace lowering_n_d_unrolled {
@@ -1100,9 +1292,8 @@ struct UnrollTransferReadConversion
     if (xferOp.getVectorType().getRank() <= options.targetRank)
       return rewriter.notifyMatchFailure(
           xferOp, "vector rank is less or equal to target rank");
-    if (isTensorOp(xferOp) && !options.lowerTensors)
-      return rewriter.notifyMatchFailure(
-          xferOp, "transfers operating on tensors are excluded");
+    if (failed(checkLowerTensors(xferOp, rewriter)))
+      return failure();
     // Transfer ops that modify the element type are not supported atm.
     if (xferOp.getVectorType().getElementType() !=
         xferOp.getShapedType().getElementType())
@@ -1238,7 +1429,7 @@ struct UnrollTransferWriteConversion
     if (inputVectorTy.getRank() <= options.targetRank)
       return failure();
 
-    if (isTensorOp(xferOp) && !options.lowerTensors)
+    if (failed(checkLowerTensors(xferOp, rewriter)))
       return failure();
     // Transfer ops that modify the element type are not supported atm.
     if (inputVectorTy.getElementType() !=
@@ -1503,7 +1694,10 @@ void mlir::populateVectorToSCFConversionPatterns(
                  lowering_n_d::TransferOpConversion<TransferWriteOp>>(
         patterns.getContext(), options);
   }
-
+  if (options.lowerScalable) {
+    patterns.add<lowering_n_d::ScalableTransposeTransferWriteConversion>(
+        patterns.getContext(), options);
+  }
   if (options.targetRank == 1) {
     patterns.add<lowering_1_d::TransferOp1dConversion<TransferReadOp>,
                  lowering_1_d::TransferOp1dConversion<TransferWriteOp>>(
@@ -1522,6 +1716,7 @@ struct ConvertVectorToSCFPass
     this->fullUnroll = options.unroll;
     this->targetRank = options.targetRank;
     this->lowerTensors = options.lowerTensors;
+    this->lowerScalable = options.lowerScalable;
   }
 
   void runOnOperation() override {
@@ -1529,6 +1724,7 @@ struct ConvertVectorToSCFPass
     options.unroll = fullUnroll;
     options.targetRank = targetRank;
     options.lowerTensors = lowerTensors;
+    options.lowerScalable = lowerScalable;
 
     // Lower permutation maps first.
     RewritePatternSet lowerTransferPatterns(&getContext());
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 11780f84697b15b..4c943ebbe3144f9 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1546,6 +1546,9 @@ LogicalResult ParallelOp::verify() {
     if (!isWrapper())
       return emitOpError() << "must take a loop wrapper role if nested inside "
                               "of 'omp.distribute'";
+    if (!isComposite())
+      return emitError()
+             << "'omp.composite' attribute missing from composite wrapper";
 
     if (LoopWrapperInterface nested = getNestedWrapper()) {
       // Check for the allowed leaf constructs that may appear in a composite
@@ -1555,6 +1558,9 @@ LogicalResult ParallelOp::verify() {
     } else {
       return emitOpError() << "must not wrap an 'omp.loop_nest' directly";
     }
+  } else if (isComposite()) {
+    return emitError()
+           << "'omp.composite' attribute present in non-composite wrapper";
   }
 
   if (getAllocateVars().size() != getAllocatorVars().size())
@@ -1748,11 +1754,28 @@ LogicalResult WsloopOp::verify() {
   if (!isWrapper())
     return emitOpError() << "must be a loop wrapper";
 
+  auto wrapper =
+      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
+  bool isCompositeChildLeaf =
+      wrapper && wrapper.isWrapper() &&
+      (!llvm::isa<ParallelOp>(wrapper) ||
+       llvm::isa_and_present<DistributeOp>(wrapper->getParentOp()));
   if (LoopWrapperInterface nested = getNestedWrapper()) {
+    if (!isComposite())
+      return emitError()
+             << "'omp.composite' attribute missing from composite wrapper";
+
     // Check for the allowed leaf constructs that may appear in a composite
     // construct directly after DO/FOR.
     if (!isa<SimdOp>(nested))
       return emitError() << "only supported nested wrapper is 'omp.simd'";
+
+  } else if (isComposite() && !isCompositeChildLeaf) {
+    return emitError()
+           << "'omp.composite' attribute present in non-composite wrapper";
+  } else if (!isComposite() && isCompositeChildLeaf) {
+    return emitError()
+           << "'omp.composite' attribute missing from composite wrapper";
   }
 
   return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
@@ -1796,6 +1819,21 @@ LogicalResult SimdOp::verify() {
   if (getNestedWrapper())
     return emitOpError() << "must wrap an 'omp.loop_nest' directly";
 
+  auto wrapper =
+      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
+  bool isCompositeChildLeaf =
+      wrapper && wrapper.isWrapper() &&
+      (!llvm::isa<ParallelOp>(wrapper) ||
+       llvm::isa_and_present<DistributeOp>(wrapper->getParentOp()));
+
+  if (!isComposite() && isCompositeChildLeaf)
+    return emitError()
+           << "'omp.composite' attribute missing from composite wrapper";
+
+  if (isComposite() && !isCompositeChildLeaf)
+    return emitError()
+           << "'omp.composite' attribute present in non-composite wrapper";
+
   return success();
 }
 
@@ -1825,11 +1863,17 @@ LogicalResult DistributeOp::verify() {
     return emitOpError() << "must be a loop wrapper";
 
   if (LoopWrapperInterface nested = getNestedWrapper()) {
+    if (!isComposite())
+      return emitError()
+             << "'omp.composite' attribute missing from composite wrapper";
     // Check for the allowed leaf constructs that may appear in a composite
     // construct directly after DISTRIBUTE.
     if (!isa<ParallelOp, SimdOp>(nested))
       return emitError() << "only supported nested wrappers are 'omp.parallel' "
                             "and 'omp.simd'";
+  } else if (isComposite()) {
+    return emitError()
+           << "'omp.composite' attribute present in non-composite wrapper";
   }
 
   return success();
@@ -2031,11 +2075,19 @@ LogicalResult TaskloopOp::verify() {
     return emitOpError() << "must be a loop wrapper";
 
   if (LoopWrapperInterface nested = getNestedWrapper()) {
+    if (!isComposite())
+      return emitError()
+             << "'omp.composite' attribute missing from composite wrapper";
+
     // Check for the allowed leaf constructs that may appear in a composite
     // construct directly after TASKLOOP.
     if (!isa<SimdOp>(nested))
       return emitError() << "only supported nested wrapper is 'omp.simd'";
+  } else if (isComposite()) {
+    return emitError()
+           << "'omp.composite' attribute present in non-composite wrapper";
   }
+
   return success();
 }
 
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index f283980ff1e0064..bd7616505348805 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -518,6 +518,23 @@ LogicalResult ModuleImport::convertLinkerOptionsMetadata() {
   return success();
 }
 
+LogicalResult ModuleImport::convertIdentMetadata() {
+  for (const llvm::NamedMDNode &named : llvmModule->named_metadata()) {
+    // llvm.ident should have a single operand. That operand is itself an
+    // MDNode with a single string operand.
+    if (named.getName() != LLVMDialect::getIdentAttrName())
+      continue;
+
+    if (named.getNumOperands() == 1)
+      if (auto *md = dyn_cast<llvm::MDNode>(named.getOperand(0)))
+        if (md->getNumOperands() == 1)
+          if (auto *mdStr = dyn_cast<llvm::MDString>(md->getOperand(0)))
+            mlirModule->setAttr(LLVMDialect::getIdentAttrName(),
+                                builder.getStringAttr(mdStr->getString()));
+  }
+  return success();
+}
+
 LogicalResult ModuleImport::convertMetadata() {
   OpBuilder::InsertionGuard guard(builder);
   builder.setInsertionPointToEnd(mlirModule.getBody());
@@ -546,6 +563,8 @@ LogicalResult ModuleImport::convertMetadata() {
   }
   if (failed(convertLinkerOptionsMetadata()))
     return failure();
+  if (failed(convertIdentMetadata()))
+    return failure();
   return success();
 }
 
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index b468228ea78b784..930300d26c44792 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1814,6 +1814,20 @@ LogicalResult ModuleTranslation::createTBAAMetadata() {
   return success();
 }
 
+LogicalResult ModuleTranslation::createIdentMetadata() {
+  if (auto attr = mlirModule->getAttrOfType<StringAttr>(
+          LLVMDialect::getIdentAttrName())) {
+    StringRef ident = attr;
+    llvm::LLVMContext &ctx = llvmModule->getContext();
+    llvm::NamedMDNode *namedMd =
+        llvmModule->getOrInsertNamedMetadata(LLVMDialect::getIdentAttrName());
+    llvm::MDNode *md = llvm::MDNode::get(ctx, llvm::MDString::get(ctx, ident));
+    namedMd->addOperand(md);
+  }
+
+  return success();
+}
+
 void ModuleTranslation::setLoopMetadata(Operation *op,
                                         llvm::Instruction *inst) {
   LoopAnnotationAttr attr =
@@ -1965,6 +1979,8 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
     return nullptr;
   if (failed(translator.createTBAAMetadata()))
     return nullptr;
+  if (failed(translator.createIdentMetadata()))
+    return nullptr;
 
   // Convert other top-level operations if possible.
   for (Operation &o : getModuleBody(module).getOperations()) {
diff --git a/mlir/test/Conversion/ConvertToSPIRV/scf.mlir b/mlir/test/Conversion/ConvertToSPIRV/scf.mlir
index 246464928b81c0c..350ebcfe57b15b7 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/scf.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/scf.mlir
@@ -33,6 +33,16 @@ func.func @if_yield(%arg0: i1) -> f32 {
 }
 
 // CHECK-LABEL: @while
+// CHECK:       spirv.mlir.loop {
+// CHECK:         spirv.Branch ^[[HEADER:.*]](%{{.*}} : i32)
+// CHECK:       ^[[HEADER]]
+// CHECK:         spirv.BranchConditional %{{.*}}, ^[[BODY:.*]](%{{.*}} : i32), ^[[MERGE:.*]]
+// CHECK:       ^[[BODY]]
+// CHECK:       spirv.Branch
+// CHECK:       ^[[MERGE]]
+// CHECK:         spirv.mlir.merge
+// CHECK:       }
+// CHECK:       spirv.Load "Function"
 func.func @while(%arg0: i32, %arg1: i32) -> i32 {
   %c2_i32 = arith.constant 2 : i32
   %0 = scf.while (%arg3 = %arg0) : (i32) -> (i32) {
@@ -45,3 +55,26 @@ func.func @while(%arg0: i32, %arg1: i32) -> i32 {
   }
   return %0 : i32
 }
+
+// CHECK-LABEL: @for
+// CHECK:       spirv.mlir.loop {
+// CHECK:         spirv.Branch ^[[HEADER:.*]](%{{.*}}, %{{.*}}, %{{.*}} : i32, f32, f32)
+// CHECK:       ^[[HEADER]]
+// CHECK:         spirv.BranchConditional %{{.*}}, ^[[BODY:.*]], ^[[MERGE:.*]]
+// CHECK:       ^[[BODY]]
+// CHECK:         spirv.Branch ^[[HEADER]]
+// CHECK:       ^[[MERGE]]
+// CHECK:         spirv.mlir.merge
+// CHECK:      }
+func.func @for() {
+  %lb = arith.constant 4 : index
+  %ub = arith.constant 42 : index
+  %step = arith.constant 2 : index
+  %s0 = arith.constant 0.0 : f32
+  %s1 = arith.constant 1.0 : f32
+  %result:2 = scf.for %i0 = %lb to %ub step %step iter_args(%si = %s0, %sj = %s1) -> (f32, f32) {
+    %sn = arith.addf %si, %si : f32
+    scf.yield %sn, %sn: f32, f32
+  }
+  return
+}
diff --git a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
index 4d0ef06d7e92f98..a93976673934295 100644
--- a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
+++ b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
@@ -6,65 +6,69 @@ module attributes {
 
 // CHECK-LABEL: @float32_unary_scalar
 func.func @float32_unary_scalar(%arg0: f32) {
+  // CHECK: spirv.GL.Atan %{{.*}}: f32
+  %0 = math.atan %arg0 : f32
   // CHECK: spirv.GL.Cos %{{.*}}: f32
-  %0 = math.cos %arg0 : f32
+  %1 = math.cos %arg0 : f32
   // CHECK: spirv.GL.Exp %{{.*}}: f32
-  %1 = math.exp %arg0 : f32
+  %2 = math.exp %arg0 : f32
   // CHECK: %[[EXP:.+]] = spirv.GL.Exp %arg0
   // CHECK: %[[ONE:.+]] = spirv.Constant 1.000000e+00 : f32
   // CHECK: spirv.FSub %[[EXP]], %[[ONE]]
-  %2 = math.expm1 %arg0 : f32
+  %3 = math.expm1 %arg0 : f32
   // CHECK: spirv.GL.Log %{{.*}}: f32
-  %3 = math.log %arg0 : f32
+  %4 = math.log %arg0 : f32
   // CHECK: %[[ONE:.+]] = spirv.Constant 1.000000e+00 : f32
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.GL.Log %[[ADDONE]]
-  %4 = math.log1p %arg0 : f32
+  %5 = math.log1p %arg0 : f32
   // CHECK: spirv.GL.RoundEven %{{.*}}: f32
-  %5 = math.roundeven %arg0 : f32
+  %6 = math.roundeven %arg0 : f32
   // CHECK: spirv.GL.InverseSqrt %{{.*}}: f32
-  %6 = math.rsqrt %arg0 : f32
+  %7 = math.rsqrt %arg0 : f32
   // CHECK: spirv.GL.Sqrt %{{.*}}: f32
-  %7 = math.sqrt %arg0 : f32
+  %8 = math.sqrt %arg0 : f32
   // CHECK: spirv.GL.Tanh %{{.*}}: f32
-  %8 = math.tanh %arg0 : f32
+  %9 = math.tanh %arg0 : f32
   // CHECK: spirv.GL.Sin %{{.*}}: f32
-  %9 = math.sin %arg0 : f32
+  %10 = math.sin %arg0 : f32
   // CHECK: spirv.GL.FAbs %{{.*}}: f32
-  %10 = math.absf %arg0 : f32
+  %11 = math.absf %arg0 : f32
   // CHECK: spirv.GL.Ceil %{{.*}}: f32
-  %11 = math.ceil %arg0 : f32
+  %12 = math.ceil %arg0 : f32
   // CHECK: spirv.GL.Floor %{{.*}}: f32
-  %12 = math.floor %arg0 : f32
+  %13 = math.floor %arg0 : f32
   return
 }
 
 // CHECK-LABEL: @float32_unary_vector
 func.func @float32_unary_vector(%arg0: vector<3xf32>) {
+  // CHECK: spirv.GL.Atan %{{.*}}: vector<3xf32>
+  %0 = math.atan %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Cos %{{.*}}: vector<3xf32>
-  %0 = math.cos %arg0 : vector<3xf32>
+  %1 = math.cos %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Exp %{{.*}}: vector<3xf32>
-  %1 = math.exp %arg0 : vector<3xf32>
+  %2 = math.exp %arg0 : vector<3xf32>
   // CHECK: %[[EXP:.+]] = spirv.GL.Exp %arg0
   // CHECK: %[[ONE:.+]] = spirv.Constant dense<1.000000e+00> : vector<3xf32>
   // CHECK: spirv.FSub %[[EXP]], %[[ONE]]
-  %2 = math.expm1 %arg0 : vector<3xf32>
+  %3 = math.expm1 %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Log %{{.*}}: vector<3xf32>
-  %3 = math.log %arg0 : vector<3xf32>
+  %4 = math.log %arg0 : vector<3xf32>
   // CHECK: %[[ONE:.+]] = spirv.Constant dense<1.000000e+00> : vector<3xf32>
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.GL.Log %[[ADDONE]]
-  %4 = math.log1p %arg0 : vector<3xf32>
+  %5 = math.log1p %arg0 : vector<3xf32>
   // CHECK: spirv.GL.RoundEven %{{.*}}: vector<3xf32>
-  %5 = math.roundeven %arg0 : vector<3xf32>
+  %6 = math.roundeven %arg0 : vector<3xf32>
   // CHECK: spirv.GL.InverseSqrt %{{.*}}: vector<3xf32>
-  %6 = math.rsqrt %arg0 : vector<3xf32>
+  %7 = math.rsqrt %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Sqrt %{{.*}}: vector<3xf32>
-  %7 = math.sqrt %arg0 : vector<3xf32>
+  %8 = math.sqrt %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Tanh %{{.*}}: vector<3xf32>
-  %8 = math.tanh %arg0 : vector<3xf32>
+  %9 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Sin %{{.*}}: vector<3xf32>
-  %9 = math.sin %arg0 : vector<3xf32>
+  %10 = math.sin %arg0 : vector<3xf32>
   return
 }
 
@@ -229,18 +233,20 @@ module attributes {
 
 // CHECK-LABEL: @vector_2d
 func.func @vector_2d(%arg0: vector<2x2xf32>) {
+  // CHECK-NEXT: math.atan {{.+}} : vector<2x2xf32>
+  %0 = math.atan %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.cos {{.+}} : vector<2x2xf32>
-  %0 = math.cos %arg0 : vector<2x2xf32>
+  %1 = math.cos %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.exp {{.+}} : vector<2x2xf32>
-  %1 = math.exp %arg0 : vector<2x2xf32>
+  %2 = math.exp %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.absf {{.+}} : vector<2x2xf32>
-  %2 = math.absf %arg0 : vector<2x2xf32>
+  %3 = math.absf %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.ceil {{.+}} : vector<2x2xf32>
-  %3 = math.ceil %arg0 : vector<2x2xf32>
+  %4 = math.ceil %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.floor {{.+}} : vector<2x2xf32>
-  %4 = math.floor %arg0 : vector<2x2xf32>
+  %5 = math.floor %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.powf {{.+}}, {{%.+}} : vector<2x2xf32>
-  %5 = math.powf %arg0, %arg0 : vector<2x2xf32>
+  %6 = math.powf %arg0, %arg0 : vector<2x2xf32>
   // CHECK-NEXT: return
   return
 }
@@ -249,18 +255,20 @@ func.func @vector_2d(%arg0: vector<2x2xf32>) {
 
 // CHECK-LABEL: @tensor_1d
 func.func @tensor_1d(%arg0: tensor<2xf32>) {
+  // CHECK-NEXT: math.atan {{.+}} : tensor<2xf32>
+  %0 = math.atan %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.cos {{.+}} : tensor<2xf32>
-  %0 = math.cos %arg0 : tensor<2xf32>
+  %1 = math.cos %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.exp {{.+}} : tensor<2xf32>
-  %1 = math.exp %arg0 : tensor<2xf32>
+  %2 = math.exp %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.absf {{.+}} : tensor<2xf32>
-  %2 = math.absf %arg0 : tensor<2xf32>
+  %3 = math.absf %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.ceil {{.+}} : tensor<2xf32>
-  %3 = math.ceil %arg0 : tensor<2xf32>
+  %4 = math.ceil %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.floor {{.+}} : tensor<2xf32>
-  %4 = math.floor %arg0 : tensor<2xf32>
+  %5 = math.floor %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.powf {{.+}}, {{%.+}} : tensor<2xf32>
-  %5 = math.powf %arg0, %arg0 : tensor<2xf32>
+  %6 = math.powf %arg0, %arg0 : tensor<2xf32>
   // CHECK-NEXT: return
   return
 }
diff --git a/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir b/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
index 03a5c2f2bc9b155..e9ca838354c0de5 100644
--- a/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
+++ b/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
@@ -4,83 +4,91 @@ module attributes { spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [Kerne
 
 // CHECK-LABEL: @float32_unary_scalar
 func.func @float32_unary_scalar(%arg0: f32) {
+  // CHECK: spirv.CL.atan %{{.*}}: f32
+  %0 = math.atan %arg0 : f32
   // CHECK: spirv.CL.cos %{{.*}}: f32
-  %0 = math.cos %arg0 : f32
+  %1 = math.cos %arg0 : f32
   // CHECK: spirv.CL.exp %{{.*}}: f32
-  %1 = math.exp %arg0 : f32
+  %2 = math.exp %arg0 : f32
   // CHECK: %[[EXP:.+]] = spirv.CL.exp %arg0
   // CHECK: %[[ONE:.+]] = spirv.Constant 1.000000e+00 : f32
   // CHECK: spirv.FSub %[[EXP]], %[[ONE]]
-  %2 = math.expm1 %arg0 : f32
+  %3 = math.expm1 %arg0 : f32
   // CHECK: spirv.CL.log %{{.*}}: f32
-  %3 = math.log %arg0 : f32
+  %4 = math.log %arg0 : f32
   // CHECK: %[[ONE:.+]] = spirv.Constant 1.000000e+00 : f32
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.CL.log %[[ADDONE]]
-  %4 = math.log1p %arg0 : f32
+  %5 = math.log1p %arg0 : f32
   // CHECK: spirv.CL.rint %{{.*}}: f32
-  %5 = math.roundeven %arg0 : f32
+  %6 = math.roundeven %arg0 : f32
   // CHECK: spirv.CL.rsqrt %{{.*}}: f32
-  %6 = math.rsqrt %arg0 : f32
+  %7 = math.rsqrt %arg0 : f32
   // CHECK: spirv.CL.sqrt %{{.*}}: f32
-  %7 = math.sqrt %arg0 : f32
+  %8 = math.sqrt %arg0 : f32
   // CHECK: spirv.CL.tanh %{{.*}}: f32
-  %8 = math.tanh %arg0 : f32
+  %9 = math.tanh %arg0 : f32
   // CHECK: spirv.CL.sin %{{.*}}: f32
-  %9 = math.sin %arg0 : f32
+  %10 = math.sin %arg0 : f32
   // CHECK: spirv.CL.fabs %{{.*}}: f32
-  %10 = math.absf %arg0 : f32
+  %11 = math.absf %arg0 : f32
   // CHECK: spirv.CL.ceil %{{.*}}: f32
-  %11 = math.ceil %arg0 : f32
+  %12 = math.ceil %arg0 : f32
   // CHECK: spirv.CL.floor %{{.*}}: f32
-  %12 = math.floor %arg0 : f32
+  %13 = math.floor %arg0 : f32
   // CHECK: spirv.CL.erf %{{.*}}: f32
-  %13 = math.erf %arg0 : f32
+  %14 = math.erf %arg0 : f32
   // CHECK: spirv.CL.round %{{.*}}: f32
-  %14 = math.round %arg0 : f32
+  %15 = math.round %arg0 : f32
   return
 }
 
 // CHECK-LABEL: @float32_unary_vector
 func.func @float32_unary_vector(%arg0: vector<3xf32>) {
+  // CHECK: spirv.CL.atan %{{.*}}: vector<3xf32>
+  %0 = math.atan %arg0 : vector<3xf32>
   // CHECK: spirv.CL.cos %{{.*}}: vector<3xf32>
-  %0 = math.cos %arg0 : vector<3xf32>
+  %1 = math.cos %arg0 : vector<3xf32>
   // CHECK: spirv.CL.exp %{{.*}}: vector<3xf32>
-  %1 = math.exp %arg0 : vector<3xf32>
+  %2 = math.exp %arg0 : vector<3xf32>
   // CHECK: %[[EXP:.+]] = spirv.CL.exp %arg0
   // CHECK: %[[ONE:.+]] = spirv.Constant dense<1.000000e+00> : vector<3xf32>
   // CHECK: spirv.FSub %[[EXP]], %[[ONE]]
-  %2 = math.expm1 %arg0 : vector<3xf32>
+  %3 = math.expm1 %arg0 : vector<3xf32>
   // CHECK: spirv.CL.log %{{.*}}: vector<3xf32>
-  %3 = math.log %arg0 : vector<3xf32>
+  %4 = math.log %arg0 : vector<3xf32>
   // CHECK: %[[ONE:.+]] = spirv.Constant dense<1.000000e+00> : vector<3xf32>
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.CL.log %[[ADDONE]]
-  %4 = math.log1p %arg0 : vector<3xf32>
+  %5 = math.log1p %arg0 : vector<3xf32>
   // CHECK: spirv.CL.rint %{{.*}}: vector<3xf32>
-  %5 = math.roundeven %arg0 : vector<3xf32>
+  %6 = math.roundeven %arg0 : vector<3xf32>
   // CHECK: spirv.CL.rsqrt %{{.*}}: vector<3xf32>
-  %6 = math.rsqrt %arg0 : vector<3xf32>
+  %7 = math.rsqrt %arg0 : vector<3xf32>
   // CHECK: spirv.CL.sqrt %{{.*}}: vector<3xf32>
-  %7 = math.sqrt %arg0 : vector<3xf32>
+  %8 = math.sqrt %arg0 : vector<3xf32>
   // CHECK: spirv.CL.tanh %{{.*}}: vector<3xf32>
-  %8 = math.tanh %arg0 : vector<3xf32>
+  %9 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.CL.sin %{{.*}}: vector<3xf32>
-  %9 = math.sin %arg0 : vector<3xf32>
+  %10 = math.sin %arg0 : vector<3xf32>
   return
 }
 
 // CHECK-LABEL: @float32_binary_scalar
 func.func @float32_binary_scalar(%lhs: f32, %rhs: f32) {
+  // CHECK: spirv.CL.atan2 %{{.*}}: f32
+  %0 = math.atan2 %lhs, %rhs : f32
   // CHECK: spirv.CL.pow %{{.*}}: f32
-  %0 = math.powf %lhs, %rhs : f32
+  %1 = math.powf %lhs, %rhs : f32
   return
 }
 
 // CHECK-LABEL: @float32_binary_vector
 func.func @float32_binary_vector(%lhs: vector<4xf32>, %rhs: vector<4xf32>) {
+  // CHECK: spirv.CL.atan2 %{{.*}}: vector<4xf32>
+  %0 = math.atan2 %lhs, %rhs : vector<4xf32>
   // CHECK: spirv.CL.pow %{{.*}}: vector<4xf32>
-  %0 = math.powf %lhs, %rhs : vector<4xf32>
+  %1 = math.powf %lhs, %rhs : vector<4xf32>
   return
 }
 
@@ -118,18 +126,20 @@ module attributes {
 
 // CHECK-LABEL: @vector_2d
 func.func @vector_2d(%arg0: vector<2x2xf32>) {
+  // CHECK-NEXT: math.atan {{.+}} : vector<2x2xf32>
+  %0 = math.atan %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.cos {{.+}} : vector<2x2xf32>
-  %0 = math.cos %arg0 : vector<2x2xf32>
+  %1 = math.cos %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.exp {{.+}} : vector<2x2xf32>
-  %1 = math.exp %arg0 : vector<2x2xf32>
+  %2 = math.exp %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.absf {{.+}} : vector<2x2xf32>
-  %2 = math.absf %arg0 : vector<2x2xf32>
+  %3 = math.absf %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.ceil {{.+}} : vector<2x2xf32>
-  %3 = math.ceil %arg0 : vector<2x2xf32>
+  %4 = math.ceil %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.floor {{.+}} : vector<2x2xf32>
-  %4 = math.floor %arg0 : vector<2x2xf32>
+  %5 = math.floor %arg0 : vector<2x2xf32>
   // CHECK-NEXT: math.powf {{.+}}, {{%.+}} : vector<2x2xf32>
-  %5 = math.powf %arg0, %arg0 : vector<2x2xf32>
+  %6 = math.powf %arg0, %arg0 : vector<2x2xf32>
   // CHECK-NEXT: return
   return
 }
@@ -138,18 +148,20 @@ func.func @vector_2d(%arg0: vector<2x2xf32>) {
 
 // CHECK-LABEL: @tensor_1d
 func.func @tensor_1d(%arg0: tensor<2xf32>) {
+  // CHECK-NEXT: math.atan {{.+}} : tensor<2xf32>
+  %0 = math.atan %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.cos {{.+}} : tensor<2xf32>
-  %0 = math.cos %arg0 : tensor<2xf32>
+  %1 = math.cos %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.exp {{.+}} : tensor<2xf32>
-  %1 = math.exp %arg0 : tensor<2xf32>
+  %2 = math.exp %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.absf {{.+}} : tensor<2xf32>
-  %2 = math.absf %arg0 : tensor<2xf32>
+  %3 = math.absf %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.ceil {{.+}} : tensor<2xf32>
-  %3 = math.ceil %arg0 : tensor<2xf32>
+  %4 = math.ceil %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.floor {{.+}} : tensor<2xf32>
-  %4 = math.floor %arg0 : tensor<2xf32>
+  %5 = math.floor %arg0 : tensor<2xf32>
   // CHECK-NEXT: math.powf {{.+}}, {{%.+}} : tensor<2xf32>
-  %5 = math.powf %arg0, %arg0 : tensor<2xf32>
+  %6 = math.powf %arg0, %arg0 : tensor<2xf32>
   // CHECK-NEXT: return
   return
 }
diff --git a/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir b/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir
index caf17bc91ced237..9ea0093eff7868d 100644
--- a/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir
+++ b/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -allow-unregistered-dialect -convert-scf-to-cf %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -convert-scf-to-cf -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: func @simple_std_for_loop(%{{.*}}: index, %{{.*}}: index, %{{.*}}: index) {
 //  CHECK-NEXT:  cf.br ^bb1(%{{.*}} : index)
@@ -675,3 +675,25 @@ func.func @forall(%num_threads: index) {
   }
   return
 }
+
+// -----
+
+//      CHECK: #loop_unroll = #llvm.loop_unroll<disable = true>
+// CHECK-NEXT: #loop_unroll1 = #llvm.loop_unroll<full = true>
+// CHECK-NEXT: #[[NO_UNROLL:.*]] = #llvm.loop_annotation<unroll = #loop_unroll>
+// CHECK-NEXT: #[[FULL_UNROLL:.*]] = #llvm.loop_annotation<unroll = #loop_unroll1>
+//      CHECK: cf.cond_br %{{.*}}, ^bb2, ^bb6 {llvm.loop_annotation = #[[NO_UNROLL]]}
+//      CHECK: cf.cond_br %{{.*}}, ^bb4, ^bb5 {llvm.loop_annotation = #[[FULL_UNROLL]]}
+#no_unroll = #llvm.loop_annotation<unroll = <disable = true>>
+#full_unroll = #llvm.loop_annotation<unroll = <full = true>>
+func.func @simple_std_for_loops_annotation(%arg0 : index, %arg1 : index, %arg2 : index) {
+  scf.for %i0 = %arg0 to %arg1 step %arg2 {
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c4 = arith.constant 4 : index
+    scf.for %i1 = %c0 to %c4 step %c1 {
+      %c1_0 = arith.constant 1 : index
+    } {llvm.loop_annotation = #full_unroll}
+  } {llvm.loop_annotation = #no_unroll}
+  return
+}
\ No newline at end of file
diff --git a/mlir/test/Conversion/VectorToSCF/tensor-transfer-ops.mlir b/mlir/test/Conversion/VectorToSCF/tensor-transfer-ops.mlir
index dac8e018f845ffd..6ec74f6b32db944 100644
--- a/mlir/test/Conversion/VectorToSCF/tensor-transfer-ops.mlir
+++ b/mlir/test/Conversion/VectorToSCF/tensor-transfer-ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{lower-tensors=true}))" -split-input-file -allow-unregistered-dialect | FileCheck %s
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{lower-tensors=true lower-scalable=true}))" -split-input-file -allow-unregistered-dialect | FileCheck %s
 
 // CHECK-LABEL: func @transfer_read_2d(
 //       CHECK: %[[ALLOC:.*]] = memref.alloca() : memref<vector<4x9xf32>>
@@ -36,3 +36,16 @@ func.func @transfer_write_2d(%A : tensor<?x?xf32>, %vec : vector<2x3xf32>,
   return %t : tensor<?x?xf32>
 }
 
+// -----
+
+// CHECK-LABEL: func @scalable_transpose_store
+//  CHECK-SAME: %[[TENSOR:[a-z0-9]+]]: tensor<?x?xf32>
+//       CHECK: %[[RESULT:.*]] = scf.for {{.*}} iter_args(%[[ITER_ARG:.*]] = %[[TENSOR]]) -> (tensor<?x?xf32>)
+//       CHECK:   %[[WRITE_SLICE:.*]] = vector.transfer_write %{{.*}} %[[ITER_ARG]]
+//       CHECK:   scf.yield %[[WRITE_SLICE]]
+//       CHECK: return %[[RESULT]]
+func.func @scalable_transpose_store(%vec: vector<4x[4]xf32>, %A: tensor<?x?xf32>, %base1: index, %base2: index) -> tensor<?x?xf32> {
+  %transpose = vector.transpose %vec, [1, 0] : vector<4x[4]xf32> to vector<[4]x4xf32>
+  %result = vector.transfer_write %transpose, %A[%base1, %base2] {in_bounds = [true, true]} : vector<[4]x4xf32>,  tensor<?x?xf32>
+  return %result : tensor<?x?xf32>
+}
diff --git a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir
index 3f4e70a6835af5f..d7620b740899257 100644
--- a/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir
+++ b/mlir/test/Conversion/VectorToSCF/vector-to-scf.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf))" -split-input-file -allow-unregistered-dialect | FileCheck %s
-// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true}))" -split-input-file -allow-unregistered-dialect | FileCheck %s --check-prefix=FULL-UNROLL
+// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf{full-unroll=true lower-scalable=true}))" -split-input-file -allow-unregistered-dialect | FileCheck %s --check-prefix=FULL-UNROLL
 // RUN: mlir-opt %s "-convert-vector-to-scf=full-unroll target-rank=0" -split-input-file -allow-unregistered-dialect | FileCheck %s --check-prefix=TARGET-RANK-ZERO
 
 // CHECK-LABEL: func @vector_transfer_ops_0d(
@@ -803,3 +803,109 @@ func.func @unroll_transfer_write_target_rank_zero(%vec : vector<2xi32>) {
 // TARGET-RANK-ZERO: %[[EXTRACTED2:.*]] = vector.extract {{.*}} : i32 from vector<2xi32>
 // TARGET-RANK-ZERO: %[[BROADCASTED2:.*]] = vector.broadcast %[[EXTRACTED2]] : i32 to vector<i32>
 // TARGET-RANK-ZERO: vector.transfer_write %[[BROADCASTED2]], %[[ALLOC]]{{.*}} : vector<i32>, memref<4xi32>
+
+// -----
+
+func.func @scalable_transpose_store_unmasked(%vec: vector<4x[4]xf32>, %dest: memref<?x?xf32>, %i: index, %j: index) {
+  %transpose = vector.transpose %vec, [1, 0] : vector<4x[4]xf32> to vector<[4]x4xf32>
+  vector.transfer_write %transpose, %dest[%i, %j] {in_bounds = [true, true]} : vector<[4]x4xf32>,  memref<?x?xf32>
+  return
+}
+// FULL-UNROLL: #[[$SLICE_MAP:.+]] = affine_map<(d0)[s0] -> (d0 + s0)>
+// FULL-UNROLL-LABEL:   func.func @scalable_transpose_store_unmasked(
+// FULL-UNROLL-SAME:                                                 %[[VEC:.*]]: vector<4x[4]xf32>,
+// FULL-UNROLL-SAME:                                                 %[[DEST:.*]]: memref<?x?xf32>,
+// FULL-UNROLL-SAME:                                                 %[[I:.*]]: index,
+// FULL-UNROLL-SAME:                                                 %[[J:.*]]: index)
+// FULL-UNROLL-DAG:       %[[C0:.*]] = arith.constant 0 : index
+// FULL-UNROLL-DAG:       %[[C1:.*]] = arith.constant 1 : index
+// FULL-UNROLL-DAG:       %[[C4:.*]] = arith.constant 4 : index
+// FULL-UNROLL:           %[[SLICE_0:.*]] = vector.extract %[[VEC]][0] : vector<[4]xf32> from vector<4x[4]xf32>
+// FULL-UNROLL:           %[[SLICE_1:.*]] = vector.extract %[[VEC]][1] : vector<[4]xf32> from vector<4x[4]xf32>
+// FULL-UNROLL:           %[[SLICE_2:.*]] = vector.extract %[[VEC]][2] : vector<[4]xf32> from vector<4x[4]xf32>
+// FULL-UNROLL:           %[[SLICE_3:.*]] = vector.extract %[[VEC]][3] : vector<[4]xf32> from vector<4x[4]xf32>
+// FULL-UNROLL:           %[[VSCALE:.*]] = vector.vscale
+// FULL-UNROLL:           %[[C4_VSCALE:.*]] = arith.muli %[[VSCALE]], %[[C4]] : index
+// FULL-UNROLL:           scf.for %[[VAL_13:.*]] = %[[C0]] to %[[C4_VSCALE]] step %[[C1]] {
+// FULL-UNROLL:             %[[SLICE_I:.*]] = affine.apply #[[$SLICE_MAP]](%[[VAL_13]]){{\[}}%[[I]]]
+// FULL-UNROLL:             %[[ELEM_0:.*]] = vector.extract %[[SLICE_0]]{{\[}}%[[VAL_13]]] : f32 from vector<[4]xf32>
+// FULL-UNROLL:             %[[ELEM_1:.*]] = vector.extract %[[SLICE_1]]{{\[}}%[[VAL_13]]] : f32 from vector<[4]xf32>
+// FULL-UNROLL:             %[[ELEM_2:.*]] = vector.extract %[[SLICE_2]]{{\[}}%[[VAL_13]]] : f32 from vector<[4]xf32>
+// FULL-UNROLL:             %[[ELEM_3:.*]] = vector.extract %[[SLICE_3]]{{\[}}%[[VAL_13]]] : f32 from vector<[4]xf32>
+// FULL-UNROLL:             %[[TRANSPOSE_SLICE:.*]] = vector.from_elements %[[ELEM_0]], %[[ELEM_1]], %[[ELEM_2]], %[[ELEM_3]] : vector<4xf32>
+// FULL-UNROLL:             vector.transfer_write %[[TRANSPOSE_SLICE]], %[[DEST]]{{\[}}%[[SLICE_I]], %[[J]]] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+
+// -----
+
+func.func @scalable_transpose_store_dynamic_mask(%vec: vector<4x[4]xf32>, %dest: memref<?x?xf32>, %i: index, %j: index, %a: index, %b: index) {
+  %transpose = vector.transpose %vec, [1, 0] : vector<4x[4]xf32> to vector<[4]x4xf32>
+  %mask = vector.create_mask %a, %b : vector<[4]x4xi1>
+  vector.transfer_write %transpose, %dest[%i, %j], %mask {in_bounds = [true, true]} : vector<[4]x4xf32>,  memref<?x?xf32>
+  return
+}
+// FULL-UNROLL-LABEL:   func.func @scalable_transpose_store_dynamic_mask(
+// FULL-UNROLL-SAME:                                                     %{{.*}}, %[[A:.*]]: index, %[[B:.*]]: index)
+// FULL-UNROLL:           %[[SLICE_MASK:.*]] = vector.create_mask %[[B]] : vector<4xi1>
+// FULL-UNROLL:           scf.for %{{.*}} to %[[A]]
+// FULL-UNROLL:             vector.transfer_write {{.*}}, %[[SLICE_MASK]]
+
+// -----
+
+func.func @scalable_transpose_store_constant_mask(%vec: vector<4x[4]xf32>, %dest: memref<?x?xf32>, %i: index, %j: index) {
+  %transpose = vector.transpose %vec, [1, 0] : vector<4x[4]xf32> to vector<[4]x4xf32>
+  %mask = vector.constant_mask [4, 3] : vector<[4]x4xi1>
+  vector.transfer_write %transpose, %dest[%i, %j], %mask {in_bounds = [true, true]} : vector<[4]x4xf32>,  memref<?x?xf32>
+  return
+}
+// FULL-UNROLL-LABEL:   func.func @scalable_transpose_store_constant_mask
+// FULL-UNROLL:           %[[C3:.*]] = arith.constant 3 : index
+// FULL-UNROLL:           %[[C4:.*]] = arith.constant 4 : index
+// FULL-UNROLL:           %[[VSCALE:.*]] = vector.vscale
+// FULL-UNROLL:           %[[C4_VSCALE:.*]] = arith.muli %[[VSCALE]], %[[C4]] : index
+// FULL-UNROLL:           %[[SLICE_MASK:.*]] = vector.create_mask %[[C3]] : vector<4xi1>
+// FULL-UNROLL:           scf.for %{{.*}} to %[[C4_VSCALE]]
+// FULL-UNROLL:             vector.transfer_write {{.*}}, %[[SLICE_MASK]]
+
+// -----
+
+/// Unsupported transpose.
+func.func @negative_scalable_transpose_store_0(%vec: vector<[4]x4xf32>, %dest: memref<?x?xf32>, %i: index, %j: index) {
+  %transpose = vector.transpose %vec, [1, 0] : vector<[4]x4xf32> to vector<4x[4]xf32>
+  vector.transfer_write %transpose, %dest[%i, %j] {in_bounds = [true, true]} : vector<4x[4]xf32>,  memref<?x?xf32>
+  return
+}
+// FULL-UNROLL-LABEL: @negative_scalable_transpose_store_0
+// FULL-UNROLL-NOT:   scf.for
+
+// -----
+
+/// Non-identity permutation map (should be lowered first).
+func.func @negative_scalable_transpose_store_1(%vec: vector<4x[4]xf32>, %dest: memref<?x?xf32>, %i: index, %j: index) {
+  %transpose = vector.transpose %vec, [1, 0] : vector<4x[4]xf32> to vector<[4]x4xf32>
+  vector.transfer_write %transpose, %dest[%i, %j] {in_bounds = [true, true], permutation_map = affine_map<(d0,d1) -> (d1, d0)> } : vector<[4]x4xf32>,  memref<?x?xf32>
+  return
+}
+// FULL-UNROLL-LABEL: @negative_scalable_transpose_store_1
+// FULL-UNROLL-NOT:   scf.for
+
+
+// -----
+
+/// Out-of-bounds dim.
+func.func @negative_scalable_transpose_store_2(%vec: vector<4x[4]xf32>, %dest: memref<?x?xf32>, %i: index, %j: index) {
+  %transpose = vector.transpose %vec, [1, 0] : vector<4x[4]xf32> to vector<[4]x4xf32>
+  vector.transfer_write %transpose, %dest[%i, %j] {in_bounds = [false, true]} : vector<[4]x4xf32>,  memref<?x?xf32>
+  return
+}
+// FULL-UNROLL-LABEL: @negative_scalable_transpose_store_2
+// FULL-UNROLL-NOT:   scf.for
+
+// -----
+
+/// Source not a vector.transpose.
+func.func @negative_scalable_transpose_store_3(%vec: vector<[4]x4xf32>, %dest: memref<?x?xf32>, %i: index, %j: index) {
+  vector.transfer_write %vec, %dest[%i, %j] {in_bounds = [true, true]} : vector<[4]x4xf32>,  memref<?x?xf32>
+  return
+}
+// FULL-UNROLL-LABEL: @negative_scalable_transpose_store_3
+// FULL-UNROLL-NOT:   scf.for
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 1d1d93f0977588b..c76b07ec94a5974 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -34,11 +34,11 @@ func.func @invalid_nested_wrapper(%lb : index, %ub : index, %step : index) {
           omp.yield
         }
         omp.terminator
-      }
+      } {omp.composite}
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
 
   return
 }
@@ -53,9 +53,9 @@ func.func @no_nested_wrapper(%lb : index, %ub : index, %step : index) {
         omp.yield
       }
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
 
   return
 }
@@ -208,9 +208,9 @@ func.func @invalid_nested_wrapper(%lb : index, %ub : index, %step : index) {
         omp.yield
       }
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
 }
 
 // -----
@@ -1963,9 +1963,9 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
         omp.yield
       }
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
   return
 }
 
@@ -2171,9 +2171,9 @@ func.func @omp_distribute_nested_wrapper(%lb: index, %ub: index, %step: index) -
         "omp.yield"() : () -> ()
       }
       "omp.terminator"() : () -> ()
-    }) : () -> ()
+    }) {omp.composite} : () -> ()
     "omp.terminator"() : () -> ()
-  }
+  } {omp.composite}
 }
 
 // -----
@@ -2383,3 +2383,165 @@ func.func @masked_arg_count_mismatch(%arg0: i32, %arg1: i32) {
     }) : (i32, i32) -> ()
   return
 }
+
+// -----
+func.func @omp_parallel_missing_composite(%lb: index, %ub: index, %step: index) -> () {
+  omp.distribute {
+    // expected-error@+1 {{'omp.composite' attribute missing from composite wrapper}}
+    omp.parallel {
+      omp.wsloop {
+        omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+          omp.yield
+        }
+        omp.terminator
+      } {omp.composite}
+      omp.terminator
+    }
+    omp.terminator
+  } {omp.composite}
+  return
+}
+
+// -----
+func.func @omp_parallel_invalid_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute present in non-composite wrapper}}
+  omp.parallel {
+    omp.wsloop {
+      omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+        omp.yield
+      }
+      omp.terminator
+    }
+    omp.terminator
+  } {omp.composite}
+  return
+}
+
+// -----
+func.func @omp_wsloop_missing_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute missing from composite wrapper}}
+  omp.wsloop {
+    omp.simd {
+      omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+        omp.yield
+      }
+      omp.terminator
+    } {omp.composite}
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @omp_wsloop_invalid_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute present in non-composite wrapper}}
+  omp.wsloop {
+    omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+      omp.yield
+    }
+    omp.terminator
+  } {omp.composite}
+  return
+}
+
+// -----
+func.func @omp_wsloop_missing_composite_2(%lb: index, %ub: index, %step: index) -> () {
+  omp.distribute {
+    omp.parallel {
+      // expected-error @below {{'omp.composite' attribute missing from composite wrapper}}
+      omp.wsloop {
+        omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+          omp.yield
+        }
+        omp.terminator
+      }
+      omp.terminator
+    } {omp.composite}
+    omp.terminator
+  } {omp.composite}
+  return
+}
+
+// -----
+func.func @omp_simd_missing_composite(%lb: index, %ub: index, %step: index) -> () {
+  omp.wsloop {
+    // expected-error @below {{'omp.composite' attribute missing from composite wrapper}}
+    omp.simd {
+      omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+        omp.yield
+      }
+      omp.terminator
+    }
+    omp.terminator
+  } {omp.composite}
+  return
+}
+
+// -----
+func.func @omp_simd_invalid_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute present in non-composite wrapper}}
+  omp.simd {
+    omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+      omp.yield
+    }
+    omp.terminator
+  } {omp.composite}
+  return
+}
+
+// -----
+func.func @omp_distribute_missing_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute missing from composite wrapper}}
+  omp.distribute {
+    omp.parallel {
+      omp.wsloop {
+        omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+          omp.yield
+        }
+        omp.terminator
+      } {omp.composite}
+      omp.terminator
+    } {omp.composite}
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @omp_distribute_invalid_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute present in non-composite wrapper}}
+  omp.distribute {
+    omp.loop_nest (%0) : index = (%lb) to (%ub) step (%step) {
+      omp.yield
+    }
+    omp.terminator
+  } {omp.composite}
+  return
+}
+
+// -----
+func.func @omp_taskloop_missing_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute missing from composite wrapper}}
+  omp.taskloop {
+    omp.simd {
+      omp.loop_nest (%i) : index = (%lb) to (%ub) step (%step)  {
+        omp.yield
+      }
+      omp.terminator
+    } {omp.composite}
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @omp_taskloop_invalid_composite(%lb: index, %ub: index, %step: index) -> () {
+  // expected-error @below {{'omp.composite' attribute present in non-composite wrapper}}
+  omp.taskloop {
+    omp.loop_nest (%i) : index = (%lb) to (%ub) step (%step)  {
+      omp.yield
+    }
+    omp.terminator
+  } {omp.composite}
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index d2924998f41b87c..9ac97e069addd2b 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -110,11 +110,32 @@ func.func @omp_parallel(%data_var : memref<i32>, %if_cond : i1, %num_threads : i
           omp.yield
         }
         omp.terminator
-      }
+      } {omp.composite}
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
+
+  // CHECK: omp.distribute
+  omp.distribute {
+    // CHECK-NEXT: omp.parallel
+    omp.parallel {
+      // CHECK-NEXT: omp.wsloop
+      omp.wsloop {
+        // CHECK-NEXT: omp.simd
+        omp.simd{
+          // CHECK-NEXT: omp.loop_nest
+          omp.loop_nest (%iv) : index = (%idx) to (%idx) step (%idx) {
+            omp.yield
+          }
+          omp.terminator
+        } {omp.composite}
+        omp.terminator
+      } {omp.composite}
+      omp.terminator
+    } {omp.composite}
+    omp.terminator
+  } {omp.composite}
 
   return
 }
@@ -434,9 +455,9 @@ func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memre
         omp.yield
       }
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }) : () -> ()
+  }) {omp.composite} : () -> ()
 
   return
 }
@@ -541,9 +562,9 @@ func.func @omp_wsloop_pretty(%lb : index, %ub : index, %step : index, %data_var
         omp.yield
       }
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
 
   return
 }
@@ -794,9 +815,9 @@ func.func @omp_distribute(%chunk_size : i32, %data_var : memref<i32>, %arg0 : i3
         omp.yield
       }
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
   return
 }
 
@@ -2101,7 +2122,7 @@ func.func @omp_task(%bool_var: i1, %i64_var: i64, %i32_var: i32, %data_var: memr
   // CHECK: omp.task allocate(%[[data_var]] : memref<i32> -> %[[data_var]] : memref<i32>)
   omp.task allocate(%data_var : memref<i32> -> %data_var : memref<i32>)
       // CHECK-SAME: final(%[[bool_var]]) if(%[[bool_var]])
-      final(%bool_var) if(%bool_var) 
+      final(%bool_var) if(%bool_var)
       // CHECK-SAME: in_reduction(@add_f32 -> %[[redn_var1]] : !llvm.ptr, byref @add_f32 -> %[[redn_var2]] : !llvm.ptr)
       in_reduction(@add_f32 -> %0 : !llvm.ptr, byref @add_f32 -> %1 : !llvm.ptr)
       // CHECK-SAME: priority(%[[i32_var]] : i32) untied
@@ -2456,9 +2477,9 @@ func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
         omp.yield
       }
       omp.terminator
-    }
+    } {omp.composite}
     omp.terminator
-  }
+  } {omp.composite}
 
   // CHECK: return
   return
@@ -2585,10 +2606,10 @@ func.func @omp_target_update_data (%if_cond : i1, %device : si32, %map1: memref<
 // CHECK-LABEL: omp_targets_is_allocatable
 // CHECK-SAME: (%[[ARG0:.*]]: !llvm.ptr, %[[ARG1:.*]]: !llvm.ptr)
 func.func @omp_targets_is_allocatable(%arg0: !llvm.ptr, %arg1: !llvm.ptr) -> () {
-  // CHECK: %[[MAP0:.*]] = omp.map.info var_ptr(%[[ARG0]] : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}  
+  // CHECK: %[[MAP0:.*]] = omp.map.info var_ptr(%[[ARG0]] : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
   %mapv1 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
   // CHECK: %[[MAP1:.*]] = omp.map.info var_ptr(%[[ARG1]] : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>) map_clauses(tofrom) capture(ByRef) members(%[[MAP0]] : [0] : !llvm.ptr) -> !llvm.ptr {name = ""}
-  %mapv2 = omp.map.info var_ptr(%arg1 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>)   map_clauses(tofrom) capture(ByRef) members(%mapv1 : [0] : !llvm.ptr) -> !llvm.ptr {name = ""}  
+  %mapv2 = omp.map.info var_ptr(%arg1 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8)>)   map_clauses(tofrom) capture(ByRef) members(%mapv1 : [0] : !llvm.ptr) -> !llvm.ptr {name = ""}
   // CHECK: omp.target map_entries(%[[MAP0]] -> {{.*}}, %[[MAP1]] -> {{.*}} : !llvm.ptr, !llvm.ptr)
   omp.target map_entries(%mapv1 -> %arg2, %mapv2 -> %arg3 : !llvm.ptr, !llvm.ptr) {
     ^bb0(%arg2: !llvm.ptr, %arg3 : !llvm.ptr):
@@ -2650,27 +2671,27 @@ func.func @omp_target_enter_update_exit_data_depend(%a: memref<?xi32>, %b: memre
 // CHECK-LABEL: omp_map_with_members
 // CHECK-SAME: (%[[ARG0:.*]]: !llvm.ptr, %[[ARG1:.*]]: !llvm.ptr, %[[ARG2:.*]]: !llvm.ptr, %[[ARG3:.*]]: !llvm.ptr, %[[ARG4:.*]]: !llvm.ptr, %[[ARG5:.*]]: !llvm.ptr)
 func.func @omp_map_with_members(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr, %arg3: !llvm.ptr, %arg4: !llvm.ptr, %arg5: !llvm.ptr) -> () {
-  // CHECK: %[[MAP0:.*]] = omp.map.info var_ptr(%[[ARG0]] : !llvm.ptr, i32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}  
+  // CHECK: %[[MAP0:.*]] = omp.map.info var_ptr(%[[ARG0]] : !llvm.ptr, i32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
   %mapv1 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  // CHECK: %[[MAP1:.*]] = omp.map.info var_ptr(%[[ARG1]] : !llvm.ptr, f32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}  
+  // CHECK: %[[MAP1:.*]] = omp.map.info var_ptr(%[[ARG1]] : !llvm.ptr, f32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
   %mapv2 = omp.map.info var_ptr(%arg1 : !llvm.ptr, f32) map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
-  
+
   // CHECK: %[[MAP2:.*]] = omp.map.info var_ptr(%[[ARG2]] : !llvm.ptr, !llvm.struct<(i32, f32)>) map_clauses(to) capture(ByRef) members(%[[MAP0]], %[[MAP1]] : [0], [1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}
-  %mapv3 = omp.map.info var_ptr(%arg2 : !llvm.ptr, !llvm.struct<(i32, f32)>)   map_clauses(to) capture(ByRef) members(%mapv1, %mapv2 : [0], [1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}  
-  
+  %mapv3 = omp.map.info var_ptr(%arg2 : !llvm.ptr, !llvm.struct<(i32, f32)>)   map_clauses(to) capture(ByRef) members(%mapv1, %mapv2 : [0], [1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}
+
   // CHECK: omp.target_enter_data map_entries(%[[MAP0]], %[[MAP1]], %[[MAP2]] : !llvm.ptr, !llvm.ptr, !llvm.ptr)
   omp.target_enter_data map_entries(%mapv1, %mapv2, %mapv3 : !llvm.ptr, !llvm.ptr, !llvm.ptr){}
 
-  // CHECK: %[[MAP3:.*]] = omp.map.info var_ptr(%[[ARG3]] : !llvm.ptr, i32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}  
+  // CHECK: %[[MAP3:.*]] = omp.map.info var_ptr(%[[ARG3]] : !llvm.ptr, i32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
   %mapv4 = omp.map.info var_ptr(%arg3 : !llvm.ptr, i32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  // CHECK: %[[MAP4:.*]] = omp.map.info var_ptr(%[[ARG4]] : !llvm.ptr, f32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}  
+  // CHECK: %[[MAP4:.*]] = omp.map.info var_ptr(%[[ARG4]] : !llvm.ptr, f32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
   %mapv5 = omp.map.info var_ptr(%arg4 : !llvm.ptr, f32) map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  
+
   // CHECK: %[[MAP5:.*]] = omp.map.info var_ptr(%[[ARG5]] : !llvm.ptr, !llvm.struct<(i32, struct<(i32, f32)>)>) map_clauses(from) capture(ByRef) members(%[[MAP3]], %[[MAP4]] : [1,0], [1,1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}
-  %mapv6 = omp.map.info var_ptr(%arg5 : !llvm.ptr, !llvm.struct<(i32, struct<(i32, f32)>)>) map_clauses(from) capture(ByRef) members(%mapv4, %mapv5 : [1,0], [1,1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}  
- 
+  %mapv6 = omp.map.info var_ptr(%arg5 : !llvm.ptr, !llvm.struct<(i32, struct<(i32, f32)>)>) map_clauses(from) capture(ByRef) members(%mapv4, %mapv5 : [1,0], [1,1] : !llvm.ptr, !llvm.ptr) -> !llvm.ptr {name = "", partial_map = true}
+
   // CHECK: omp.target_exit_data map_entries(%[[MAP3]], %[[MAP4]], %[[MAP5]] : !llvm.ptr, !llvm.ptr, !llvm.ptr)
   omp.target_exit_data map_entries(%mapv4, %mapv5, %mapv6 : !llvm.ptr, !llvm.ptr, !llvm.ptr){}
 
diff --git a/mlir/test/Target/LLVMIR/Import/ident.ll b/mlir/test/Target/LLVMIR/Import/ident.ll
new file mode 100644
index 000000000000000..1728fdf249ede75
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/ident.ll
@@ -0,0 +1,6 @@
+; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s
+
+; CHECK: module attributes {
+; CHECK-SAME: llvm.ident = "flang version 61.7.4"
+!llvm.ident = !{!0}
+!0 = !{!"flang version 61.7.4"}
diff --git a/mlir/test/Target/LLVMIR/ident.mlir b/mlir/test/Target/LLVMIR/ident.mlir
new file mode 100644
index 000000000000000..e2c343df3c7f339
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/ident.mlir
@@ -0,0 +1,6 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK: !llvm.ident = !{![[ID:[0-9]+]]}
+// CHECK: ![[ID]] = !{!"flang version 61.7.4"}
+module attributes {llvm.ident = "flang version 61.7.4"} {
+}
diff --git a/mlir/tools/mlir-tblgen/OmpOpGen.cpp b/mlir/tools/mlir-tblgen/OmpOpGen.cpp
index 51eb43f322e6ad9..ffa2e17cc8f9168 100644
--- a/mlir/tools/mlir-tblgen/OmpOpGen.cpp
+++ b/mlir/tools/mlir-tblgen/OmpOpGen.cpp
@@ -73,12 +73,11 @@ static StringRef extractOmpClauseName(Record *clause) {
 static bool verifyArgument(DagInit *arguments, StringRef argName,
                            Init *argInit) {
   auto range = zip_equal(arguments->getArgNames(), arguments->getArgs());
-  return std::find_if(
-             range.begin(), range.end(),
-             [&](std::tuple<llvm::StringInit *const &, llvm::Init *const &> v) {
-               return std::get<0>(v)->getAsUnquotedString() == argName &&
-                      std::get<1>(v) == argInit;
-             }) != range.end();
+  return llvm::any_of(
+      range, [&](std::tuple<llvm::StringInit *const &, llvm::Init *const &> v) {
+        return std::get<0>(v)->getAsUnquotedString() == argName &&
+               std::get<1>(v) == argInit;
+      });
 }
 
 /// Check that the given string record value, identified by its name \c value,
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 4e0082aebf90c17..f2277d99b4ecb29 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3268,6 +3268,20 @@ gentbl_cc_library(
             ],
             "include/mlir/Dialect/Mesh/IR/MeshAttributes.cpp.inc",
         ),
+        (
+            [
+                "-gen-typedef-decls",
+		"-typedefs-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshTypes.h.inc",
+        ),
+        (
+            [
+                "-gen-typedef-defs",
+		"-typedefs-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshTypes.cpp.inc",
+        ),
     ],
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Dialect/Mesh/IR/MeshOps.td",
@@ -3308,6 +3322,26 @@ cc_library(
         ":MeshDialect",
         ":MeshShardingInterfaceIncGen",
         ":Support",
+        ":TensorDialect",
+        "//llvm:Support",
+    ],
+)
+
+cc_library(
+    name = "TensorShardingInterfaceImpl",
+    srcs = ["lib/Dialect/Mesh/Interfaces/TensorShardingInterfaceImpl.cpp"],
+    hdrs = [
+        "include/mlir/Dialect/Mesh/IR/TensorShardingInterfaceImpl.h",
+    ],
+    includes = ["include"],
+    deps = [
+        ":DialectUtils",
+        ":IR",
+        ":MeshDialect",
+        ":MeshShardingInterfaceIncGen",
+        ":MeshShardingInterface",
+        ":Support",
+        ":TensorDialect",
         "//llvm:Support",
     ],
 )
@@ -4826,6 +4860,7 @@ cc_library(
         ":BuiltinToLLVMIRTranslation",
         ":ComplexToLLVM",
         ":ControlFlowToLLVM",
+	":DLTITransformOps",
         ":FuncExtensions",
         ":FuncToLLVM",
         ":FuncTransformOps",
@@ -8217,6 +8252,7 @@ cc_library(
         ":ControlFlowDialect",
         ":ConversionPassIncGen",
         ":IR",
+        ":LLVMDialect",
         ":SCFDialect",
         ":SCFTransforms",
         ":TransformUtils",
@@ -9524,6 +9560,7 @@ cc_library(
         ":SparseTensorTransforms",
         ":TensorDialect",
         ":TensorInferTypeOpInterfaceImpl",
+	":TensorShardingInterfaceImpl",
         ":TensorTilingInterfaceImpl",
         ":TensorTransformOps",
         ":TensorTransforms",
@@ -11767,6 +11804,7 @@ cc_library(
         ":ArithDialect",
         ":ConversionPassIncGen",
         ":IR",
+        ":LLVMDialect",
         ":MemRefDialect",
         ":Pass",
         ":SCFDialect",
@@ -11775,6 +11813,7 @@ cc_library(
         ":Transforms",
         ":VectorDialect",
         ":VectorTransforms",
+        ":VectorUtils",
     ],
 )
 
@@ -13988,6 +14027,55 @@ cc_library(
     ],
 )
 
+td_library(
+    name = "DLTITransformTdFiles",
+    srcs = [
+        "include/mlir/Dialect/DLTI/TransformOps/DLTITransformOps.td",
+    ],
+    includes = ["include"],
+    deps = [":TransformDialectTdFiles"],
+)
+
+gentbl_cc_library(
+    name = "DLTITransformOpsIncGen",
+    tbl_outs = [
+        (
+            [
+                "-gen-op-decls",
+                "-dialect=dlti",
+            ],
+            "include/mlir/Dialect/DLTI/TransformOps/DLTITransformOps.h.inc",
+        ),
+        (
+            [
+                "-gen-op-defs",
+                "-dialect=dlti",
+            ],
+            "include/mlir/Dialect/DLTI/TransformOps/DLTITransformOps.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/DLTI/TransformOps/DLTITransformOps.td",
+    deps = [
+        ":DLTITransformTdFiles",
+    ],
+)
+
+cc_library(
+    name = "DLTITransformOps",
+    srcs = glob(["lib/Dialect/DLTI/TransformOps/*.cpp"]),
+    hdrs = glob(["include/mlir/Dialect/DLTI/TransformOps/*.h"]),
+    includes = ["include"],
+    deps = [
+        ":DataLayoutInterfaces",
+	":DLTIDialect",
+	":DLTITransformOpsIncGen",
+	":TransformDialect",
+	":TransformDialectInterfaces",
+	":TransformDialectUtils",
+    ],
+)
+
 gentbl_cc_library(
     name = "ReducerIncGen",
     tbl_outs = [