From 494db3816b0ece5b6722054f75cc2622ae1b840a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 6 Jan 2021 11:51:52 +0000
Subject: [PATCH 01/12] [LoopDeletion] Also consider loops with subloops for
 deletion.

Currently, LoopDeletion does skip loops that have sub-loops, but this
means we currently fail to remove some no-op loops.

One example are inner loops with live-out values. Those cannot be
removed by itself. But the containing loop may itself be a no-op and the
whole loop-nest can be deleted.

The legality checks do not seem to rely on analyzing inner-loops only
for correctness.

With LoopDeletion being a LoopPass, the change means that we now
unfortunately need to do some extra work in parent loops, by checking
some conditions we already checked. But there appears to be no
noticeable compile time impact:
http://llvm-compile-time-tracker.com/compare.php?from=02d11f3cda2ab5b8bf4fc02639fd1f4b8c45963e&to=843201e9cf3b6871e18c52aede5897a22994c36c&stat=instructions

This changes patch leads to ~10 more loops being deleted on
MultiSource, SPEC2000, SPEC2006 with -O3 & LTO

This patch is also required (together with a few others) to eliminate a
no-op loop in omnetpp as discussed on llvm-dev 'LoopDeletion / removal of
empty loops.' (http://lists.llvm.org/pipermail/llvm-dev/2020-December/147462.html)

This change becomes relevant after removing potentially infinite loops
is made possible in 'must-progress' loops (D86844).

Note that I added a function call with side-effects to an outer loop in
`llvm/test/Transforms/LoopDeletion/update-scev.ll` to preserve the
original spirit of the test.

Reviewed By: reames

Differential Revision: https://reviews.llvm.org/D93716
---
 llvm/lib/Transforms/Scalar/LoopDeletion.cpp   |  7 ------
 .../LoopDeletion/noop-loops-with-subloops.ll  | 19 +-------------
 .../LoopDeletion/unreachable-loops.ll         | 25 +++++++------------
 .../Transforms/LoopDeletion/update-scev.ll    |  3 +++
 4 files changed, 13 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
index 814cfc7ac6a9bc..a94676eadeabac 100644
--- a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
@@ -156,13 +156,6 @@ static LoopDeletionResult deleteLoopIfDead(Loop *L, DominatorTree &DT,
         << "Deletion requires Loop with preheader and dedicated exits.\n");
     return LoopDeletionResult::Unmodified;
   }
-  // We can't remove loops that contain subloops.  If the subloops were dead,
-  // they would already have been removed in earlier executions of this pass.
-  if (L->begin() != L->end()) {
-    LLVM_DEBUG(dbgs() << "Loop contains subloops.\n");
-    return LoopDeletionResult::Unmodified;
-  }
-
 
   BasicBlock *ExitBlock = L->getUniqueExitBlock();
 
diff --git a/llvm/test/Transforms/LoopDeletion/noop-loops-with-subloops.ll b/llvm/test/Transforms/LoopDeletion/noop-loops-with-subloops.ll
index 464c12f453a701..b7a921a8dd513a 100644
--- a/llvm/test/Transforms/LoopDeletion/noop-loops-with-subloops.ll
+++ b/llvm/test/Transforms/LoopDeletion/noop-loops-with-subloops.ll
@@ -10,24 +10,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 define void @test1(i64 %N, i64 %M, %pair_t* %ptr) willreturn {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
-; CHECK:       outer.header:
-; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
-; CHECK-NEXT:    br label [[INNER:%.*]]
-; CHECK:       inner:
-; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, [[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], [[INNER]] ]
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [[PAIR_T:%.*]], %pair_t* [[PTR:%.*]], i64 [[INNER_IV]]
-; CHECK-NEXT:    [[P:%.*]] = load [[PAIR_T]], %pair_t* [[GEP]], align 4
-; CHECK-NEXT:    [[V_0:%.*]] = extractvalue [[PAIR_T]] [[P]], 0
-; CHECK-NEXT:    [[V_1:%.*]] = extractvalue [[PAIR_T]] [[P]], 1
-; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp ult i64 [[V_0]], [[V_1]]
-; CHECK-NEXT:    [[INNER_IV_NEXT]] = add i64 [[INNER_IV]], 1
-; CHECK-NEXT:    br i1 [[INNER_EC]], label [[OUTER_LATCH]], label [[INNER]]
-; CHECK:       outer.latch:
-; CHECK-NEXT:    [[LCSSA:%.*]] = phi i64 [ [[V_1]], [[INNER]] ]
-; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp ult i64 [[OUTER_IV]], [[LCSSA]]
-; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1
-; CHECK-NEXT:    br i1 [[OUTER_EC]], label [[EXIT:%.*]], label [[OUTER_HEADER]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll b/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
index a74ddf99285e23..c9e178fbf586f8 100644
--- a/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
+++ b/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
@@ -244,22 +244,15 @@ exit:
 
 ; Delete a loop (L2) which has subloop (L3).
 ; Here we delete loop L2, but leave L3 as is.
-; FIXME: Can delete L3 as well, by iteratively going backward through the single
-; predecessor of L3 until we reach L1's block that guarantees L3 is never
-; executed.
 define void @test9(i64 %n) {
 ; CHECK-LABEL: test9
-; CHECK-LABEL: L2.preheader:
-; CHECK-NEXT: br label %L3.preheader
-; CHECK-NOT: L2:
-; CHECK-LABEL: L3.preheader:
-; CHECK-NEXT: %y.L2.lcssa = phi i64 [ undef, %L2.preheader ]
-; CHECK-NEXT: br label %L3
-; CHECK-LABEL: L3:
-; CHECK: br i1 %cond2, label %L3, label %L1.loopexit
+; CHECK-LABEL: entry:
+; CHECK-NEXT:    br label %exit
+; CHECK-LABEL: exit:
+; CHECK-NEXT:    ret  void
 ; REMARKS-LABEL: Function: test9
 ; REMARKS: Loop deleted because it never executes
-entry: 
+entry:
   br label %L1
 
 L1:
@@ -283,12 +276,12 @@ exit:
 ; We cannot delete L3 because of call within it.
 ; Since L3 is not deleted, and entirely contained within L2, L2 is also not
 ; deleted.
-; FIXME: We can delete unexecutable loops having
-; subloops contained entirely within them.
 define void @test10(i64 %n) {
 ; CHECK-LABEL: test10
-; CHECK: L2:
-; CHECK: L3:
+; CHECK-LABEL: entry:
+; CHECK-NEXT:   br label %exit
+; CHECK-LABEL: exit:
+; CHECK-NEXT:    ret void
 entry: 
   br label %L1
 
diff --git a/llvm/test/Transforms/LoopDeletion/update-scev.ll b/llvm/test/Transforms/LoopDeletion/update-scev.ll
index 44d23aa4060d1c..16a5530219177d 100644
--- a/llvm/test/Transforms/LoopDeletion/update-scev.ll
+++ b/llvm/test/Transforms/LoopDeletion/update-scev.ll
@@ -48,6 +48,7 @@ for.body6:                                        ; preds = %for.body6, %for.bod
 
 for.inc11:                                        ; preds = %for.body6
   %and.lcssa = phi i32 [ %and, %for.body6 ]
+  call void @sideeffect(i32 %and.lcssa)
   %inc12 = add nsw i32 %val, 1
   %tobool = icmp eq i32 %inc12, 0
   br i1 %tobool, label %for.cond14, label %for.body
@@ -56,6 +57,8 @@ for.cond14:                                       ; preds = %for.cond14, %for.in
   br i1 undef, label %for.cond, label %for.cond14
 }
 
+declare void @sideeffect(i32)
+
 ; LoopDeletion removes the loop %for.body7.1. Make sure %inc.lcssa.1 in the loop
 ; exit block is correctly invalidated.
 

From 0c41b1c9f93c09966b87126820d3cf41d8eebbf9 Mon Sep 17 00:00:00 2001
From: Yvan Roux <yvan.roux@linaro.org>
Date: Wed, 6 Jan 2021 16:01:38 +0100
Subject: [PATCH 02/12] [Driver][MachineOutliner] Support outlining option with
 LTO

This patch propagates the -moutline flag when LTO is enabled and avoids
passing it explicitly to the linker plugin.

Differential Revision: https://reviews.llvm.org/D93385
---
 clang/lib/Driver/ToolChains/Clang.cpp      | 21 +------------
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 36 ++++++++++++++++++++++
 clang/lib/Driver/ToolChains/CommonArgs.h   |  4 +++
 clang/test/Driver/arm-machine-outliner.c   |  9 ++++++
 4 files changed, 50 insertions(+), 20 deletions(-)
 create mode 100644 clang/test/Driver/arm-machine-outliner.c

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f8b9bf25373e88..917601836c0a8e 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6396,26 +6396,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                     options::OPT_fno_cxx_static_destructors, true))
     CmdArgs.push_back("-fno-c++-static-destructors");
 
-  if (Arg *A = Args.getLastArg(options::OPT_moutline,
-                               options::OPT_mno_outline)) {
-    if (A->getOption().matches(options::OPT_moutline)) {
-      // We only support -moutline in AArch64 and ARM targets right now. If
-      // we're not compiling for these, emit a warning and ignore the flag.
-      // Otherwise, add the proper mllvm flags.
-      if (!(Triple.isARM() || Triple.isThumb() ||
-            Triple.getArch() == llvm::Triple::aarch64 ||
-            Triple.getArch() == llvm::Triple::aarch64_32)) {
-        D.Diag(diag::warn_drv_moutline_unsupported_opt) << Triple.getArchName();
-      } else {
-        CmdArgs.push_back("-mllvm");
-        CmdArgs.push_back("-enable-machine-outliner");
-      }
-    } else {
-      // Disable all outlining behaviour.
-      CmdArgs.push_back("-mllvm");
-      CmdArgs.push_back("-enable-machine-outliner=never");
-    }
-  }
+  addMachineOutlinerArgs(D, Args, CmdArgs, Triple, /*IsLTO=*/false);
 
   if (Arg *A = Args.getLastArg(options::OPT_moutline_atomics,
                                options::OPT_mno_outline_atomics)) {
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index fe5e7536d38054..6a95aa5ec62878 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -624,6 +624,9 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
 
   // Handle remarks hotness/threshold related options.
   renderRemarksHotnessOptions(Args, CmdArgs);
+
+  addMachineOutlinerArgs(D, Args, CmdArgs, ToolChain.getEffectiveTriple(),
+                         /*IsLTO=*/true);
 }
 
 void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args,
@@ -1586,3 +1589,36 @@ unsigned tools::getOrCheckAMDGPUCodeObjectVersion(
   }
   return CodeObjVer;
 }
+
+void tools::addMachineOutlinerArgs(const Driver &D,
+                                   const llvm::opt::ArgList &Args,
+                                   llvm::opt::ArgStringList &CmdArgs,
+                                   const llvm::Triple &Triple, bool IsLTO) {
+  auto addArg = [&, IsLTO](const Twine &Arg) {
+    if (IsLTO) {
+      CmdArgs.push_back(Args.MakeArgString("-plugin-opt=" + Arg));
+    } else {
+      CmdArgs.push_back("-mllvm");
+      CmdArgs.push_back(Args.MakeArgString(Arg));
+    }
+  };
+
+  if (Arg *A = Args.getLastArg(options::OPT_moutline,
+                               options::OPT_mno_outline)) {
+    if (A->getOption().matches(options::OPT_moutline)) {
+      // We only support -moutline in AArch64 and ARM targets right now. If
+      // we're not compiling for these, emit a warning and ignore the flag.
+      // Otherwise, add the proper mllvm flags.
+      if (!(Triple.isARM() || Triple.isThumb() ||
+            Triple.getArch() == llvm::Triple::aarch64 ||
+            Triple.getArch() == llvm::Triple::aarch64_32)) {
+        D.Diag(diag::warn_drv_moutline_unsupported_opt) << Triple.getArchName();
+      } else {
+        addArg(Twine("-enable-machine-outliner"));
+      }
+    } else {
+      // Disable all outlining behaviour.
+      addArg(Twine("-enable-machine-outliner=never"));
+    }
+  }
+}
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
index 9a365f3760228d..187c340d1c3c30 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -141,6 +141,10 @@ void addX86AlignBranchArgs(const Driver &D, const llvm::opt::ArgList &Args,
 unsigned getOrCheckAMDGPUCodeObjectVersion(const Driver &D,
                                            const llvm::opt::ArgList &Args,
                                            bool Diagnose = false);
+
+void addMachineOutlinerArgs(const Driver &D, const llvm::opt::ArgList &Args,
+                            llvm::opt::ArgStringList &CmdArgs,
+                            const llvm::Triple &Triple, bool IsLTO);
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/test/Driver/arm-machine-outliner.c b/clang/test/Driver/arm-machine-outliner.c
new file mode 100644
index 00000000000000..6ac82838323a96
--- /dev/null
+++ b/clang/test/Driver/arm-machine-outliner.c
@@ -0,0 +1,9 @@
+// REQUIRES: arm-registered-target
+// RUN: %clang -target armv7-linux-gnueabihf -moutline -c %s -### 2>&1 | FileCheck %s -check-prefix=ON
+// ON: "-mllvm" "-enable-machine-outliner"
+// RUN: %clang -target armv7-linux-gnueabihf -flto -moutline %s -### 2>&1 | FileCheck %s -check-prefix=ON-LTO
+// ON-LTO: "-plugin-opt=-enable-machine-outliner"
+// RUN: %clang -target armv7-linux-gnueabihf -moutline -mno-outline -c %s -### 2>&1 | FileCheck %s -check-prefix=OFF
+// OFF: "-mllvm" "-enable-machine-outliner=never"
+// RUN: %clang -target armv7-linux-gnueabihf -flto -moutline -mno-outline %s -### 2>&1 | FileCheck %s -check-prefix=OFF-LTO
+// OFF-LTO: "-plugin-opt=-enable-machine-outliner=never"

From 3fa6cedb6be809092f8a8b27e63bd4f6dc526a08 Mon Sep 17 00:00:00 2001
From: Erich Keane <erich.keane@intel.com>
Date: Wed, 6 Jan 2021 06:46:01 -0800
Subject: [PATCH 03/12] Fix MaterializeTemporaryExpr's type when its an
 incomplete array.

Like the VarDecl that gets its type updated based on an init-list, this
patch corrects the MaterializeTemporaryExpr's type to make sure it isn't
creating an incomplete type, which leads to a handful of CodeGen crashes
(see PR 47636).

Based on @rsmith 's comments on D88236

Differential Revision: https://reviews.llvm.org/D88298
---
 clang/lib/Sema/SemaInit.cpp       | 14 +++++++++++++-
 clang/test/AST/pr47636.cpp        | 26 ++++++++++++++++++++++++++
 clang/test/CodeGenCXX/pr47636.cpp | 12 ++++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/AST/pr47636.cpp

diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index b5f31bf403d449..38f6a5975ea3ce 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -8200,9 +8200,21 @@ ExprResult InitializationSequence::Perform(Sema &S,
       if (S.CheckExceptionSpecCompatibility(CurInit.get(), DestType))
         return ExprError();
 
+      QualType MTETy = Step->Type;
+
+      // When this is an incomplete array type (such as when this is
+      // initializing an array of unknown bounds from an init list), use THAT
+      // type instead so that we propogate the array bounds.
+      if (MTETy->isIncompleteArrayType() &&
+          !CurInit.get()->getType()->isIncompleteArrayType() &&
+          S.Context.hasSameType(
+              MTETy->getPointeeOrArrayElementType(),
+              CurInit.get()->getType()->getPointeeOrArrayElementType()))
+        MTETy = CurInit.get()->getType();
+
       // Materialize the temporary into memory.
       MaterializeTemporaryExpr *MTE = S.CreateMaterializeTemporaryExpr(
-          Step->Type, CurInit.get(), Entity.getType()->isLValueReferenceType());
+          MTETy, CurInit.get(), Entity.getType()->isLValueReferenceType());
       CurInit = MTE;
 
       // If we're extending this temporary to automatic storage duration -- we
diff --git a/clang/test/AST/pr47636.cpp b/clang/test/AST/pr47636.cpp
new file mode 100644
index 00000000000000..29d2a0a06e7916
--- /dev/null
+++ b/clang/test/AST/pr47636.cpp
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -fsyntax-only %s -ast-dump | FileCheck %s
+
+int(&&intu_rvref)[] {1,2,3,4};
+// CHECK: VarDecl 0x[[GLOB_ADDR:[0-9a-f]+]] {{.*}} intu_rvref 'int (&&)[4]' listinit
+// CHECK-NEXT: ExprWithCleanups {{.*}} 'int [4]' xvalue
+// CHECK-NEXT: MaterializeTemporaryExpr {{.*}} 'int [4]' xvalue extended by Var 0x[[GLOB_ADDR]] 'intu_rvref' 'int (&&)[4]'
+// CHECK-NEXT: InitListExpr {{.*}} 'int [4]'
+
+// CHECK: FunctionDecl {{.*}} static_const
+void static_const() {
+  static const int(&&intu_rvref)[] {1,2,3,4};
+  // CHECK: VarDecl 0x[[STATIC_ADDR:[0-9a-f]+]] {{.*}} intu_rvref 'const int (&&)[4]' static listinit
+  // CHECK-NEXT: ExprWithCleanups {{.*}} 'const int [4]' xvalue
+  // CHECK-NEXT: MaterializeTemporaryExpr {{.*}} 'const int [4]' xvalue extended by Var 0x[[STATIC_ADDR]] 'intu_rvref' 'const int (&&)[4]'
+  // CHECK-NEXT: InitListExpr {{.*}} 'const int [4]'
+}
+
+// CHECK: FunctionDecl {{.*}} const_expr
+constexpr int const_expr() {
+  int(&&intu_rvref)[]{1, 2, 3, 4};
+  // CHECK: VarDecl 0x[[CE_ADDR:[0-9a-f]+]] {{.*}} intu_rvref 'int (&&)[4]' listinit
+  // CHECK-NEXT: ExprWithCleanups {{.*}} 'int [4]' xvalue
+  // CHECK-NEXT: MaterializeTemporaryExpr {{.*}} 'int [4]' xvalue extended by Var 0x[[CE_ADDR]] 'intu_rvref' 'int (&&)[4]'
+  // CHECK-NEXT: InitListExpr {{.*}} 'int [4]'
+  return intu_rvref[0];
+}
diff --git a/clang/test/CodeGenCXX/pr47636.cpp b/clang/test/CodeGenCXX/pr47636.cpp
index 64fb44114bd2bb..b6b31d62376103 100644
--- a/clang/test/CodeGenCXX/pr47636.cpp
+++ b/clang/test/CodeGenCXX/pr47636.cpp
@@ -8,3 +8,15 @@ void foo() {
   // CHECK: @_ZZ3foovE10intu_rvref = internal constant [4 x i32]* @_ZGRZ3foovE10intu_rvref_
   // CHECK: @_ZGRZ3foovE10intu_rvref_ = internal constant [4 x i32] [i32 1, i32 2, i32 3, i32 4]
 }
+
+// Example given on review, ensure this doesn't crash as well.
+constexpr int f() {
+  // CHECK: i32 @_Z1fv()
+  int(&&intu_rvref)[]{1, 2, 3, 4};
+  // CHECK: %{{.*}} = alloca [4 x i32]*
+  return intu_rvref[2];
+}
+
+void use_f() {
+  int i = f();
+}

From 25c78de6d2a50d6f90fd6cd3f0010eb3df157a6c Mon Sep 17 00:00:00 2001
From: Lei Zhang <antiagainst@google.com>
Date: Wed, 6 Jan 2021 10:28:01 -0500
Subject: [PATCH 04/12] [mlir][spirv] Update pass docs

Reviewed By: hanchung

Differential Revision: https://reviews.llvm.org/D94174
---
 mlir/include/mlir/Conversion/Passes.td | 38 +++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 2dc438534a4482..6a6ba6bbb3717b 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -144,6 +144,18 @@ def ConvertGpuOpsToROCDLOps : Pass<"convert-gpu-to-rocdl", "gpu::GPUModuleOp"> {
 
 def ConvertGPUToSPIRV : Pass<"convert-gpu-to-spirv", "ModuleOp"> {
   let summary = "Convert GPU dialect to SPIR-V dialect";
+  let description = [{
+    This pass converts supported GPU device ops to SPIR-V ops. It does not
+    handle GPU host ops.
+
+    A `gpu.func` op can have parameters to pass in resources. But in SPIR-V
+    entry functions cannot take parameters; they use descriptors to access
+    resources. By default, parameters to a `gpu.func` op will be converted to
+    global variables. These global variables will be assigned sequential binding
+    numbers following their order in the original `gpu.func` op, starting from
+    0, in set 0. One can attach `spv.interface_var_abi` to those parameters
+    to control the set and binding if wanted.
+  }];
   let constructor = "mlir::createConvertGPUToSPIRVPass()";
   let dependentDialects = ["spirv::SPIRVDialect"];
 }
@@ -155,6 +167,9 @@ def ConvertGPUToSPIRV : Pass<"convert-gpu-to-spirv", "ModuleOp"> {
 def ConvertGpuLaunchFuncToVulkanLaunchFunc
     : Pass<"convert-gpu-launch-to-vulkan-launch", "ModuleOp"> {
   let summary = "Convert gpu.launch_func to vulkanLaunch external call";
+  let description = [{
+    This pass is only intended for the mlir-vulkan-runner.
+  }];
   let constructor = "mlir::createConvertGpuLaunchFuncToVulkanLaunchFuncPass()";
   let dependentDialects = ["spirv::SPIRVDialect"];
 }
@@ -163,6 +178,9 @@ def ConvertVulkanLaunchFuncToVulkanCalls
     : Pass<"launch-func-to-vulkan", "ModuleOp"> {
   let summary = "Convert vulkanLaunch external call to Vulkan runtime external "
                 "calls";
+  let description = [{
+    This pass is only intended for the mlir-vulkan-runner.
+  }];
   let constructor = "mlir::createConvertVulkanLaunchFuncToVulkanCallsPass()";
   let dependentDialects = ["LLVM::LLVMDialect"];
 }
@@ -194,7 +212,11 @@ def ConvertLinalgToStandard : Pass<"convert-linalg-to-std", "ModuleOp"> {
 //===----------------------------------------------------------------------===//
 
 def ConvertLinalgToSPIRV : Pass<"convert-linalg-to-spirv", "ModuleOp"> {
-  let summary = "Convert Linalg ops to SPIR-V ops";
+  let summary = "Convert Linalg dialect to SPIR-V dialect";
+  let description = [{
+    This pass converts supported Linalg ops to SPIR-V ops. It's quite
+    experimental and are expected to migrate to other proper conversions.
+  }];
   let constructor = "mlir::createLinalgToSPIRVPass()";
   let dependentDialects = ["spirv::SPIRVDialect"];
 }
@@ -312,6 +334,10 @@ def ConvertShapeConstraints: Pass<"convert-shape-constraints", "FuncOp"> {
 
 def ConvertSPIRVToLLVM : Pass<"convert-spirv-to-llvm", "ModuleOp"> {
   let summary = "Convert SPIR-V dialect to LLVM dialect";
+  let description = [{
+    See https://mlir.llvm.org/docs/SPIRVToLLVMDialectConversion/
+    for more details.
+  }];
   let constructor = "mlir::createConvertSPIRVToLLVMPass()";
   let dependentDialects = ["LLVM::LLVMDialect"];
 }
@@ -375,12 +401,17 @@ def ConvertStandardToLLVM : Pass<"convert-std-to-llvm", "ModuleOp"> {
 
 def LegalizeStandardForSPIRV : Pass<"legalize-std-for-spirv"> {
   let summary = "Legalize standard ops for SPIR-V lowering";
+  let description = [{
+    The pass contains certain intra standard op conversions that are meant for
+    lowering to SPIR-V ops, e.g., folding subviews loads/stores to the original
+    loads/stores from/to the original memref.
+  }];
   let constructor = "mlir::createLegalizeStdOpsForSPIRVLoweringPass()";
   let dependentDialects = ["spirv::SPIRVDialect"];
 }
 
 def ConvertStandardToSPIRV : Pass<"convert-std-to-spirv", "ModuleOp"> {
-  let summary = "Convert Standard Ops to SPIR-V dialect";
+  let summary = "Convert Standard dialect to SPIR-V dialect";
   let constructor = "mlir::createConvertStandardToSPIRVPass()";
   let dependentDialects = ["spirv::SPIRVDialect"];
 }
@@ -459,8 +490,7 @@ def ConvertVectorToROCDL : Pass<"convert-vector-to-rocdl", "ModuleOp"> {
 //===----------------------------------------------------------------------===//
 
 def ConvertVectorToSPIRV : Pass<"convert-vector-to-spirv", "ModuleOp"> {
-  let summary = "Lower the operations from the vector dialect into the SPIR-V "
-                "dialect";
+  let summary = "Convert Vector dialect to SPIR-V dialect";
   let constructor = "mlir::createConvertVectorToSPIRVPass()";
   let dependentDialects = ["spirv::SPIRVDialect"];
 }

From 46975b5b29e2ecbf97eb7be2b124d94f0ce4b45e Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <kparzysz@quicinc.com>
Date: Wed, 6 Jan 2021 09:11:04 -0600
Subject: [PATCH 05/12] [Hexagon] Wrap functions only used in asserts in ifndef
 NDEBUG

---
 llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index 05269d37f812b2..01fd8a9ef9ce16 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -117,8 +117,11 @@ class HexagonVectorCombine {
   const HexagonSubtarget &HST;
 
 private:
+#ifndef NDEBUG
+  // These two functions are only used for assertions at the moment.
   bool isByteVecTy(Type *Ty) const;
-  bool isSectorTy(Type *Ty) const LLVM_ATTRIBUTE_UNUSED;
+  bool isSectorTy(Type *Ty) const;
+#endif
   Value *getElementRange(IRBuilder<> &Builder, Value *Lo, Value *Hi, int Start,
                          int Length) const;
 };
@@ -1406,6 +1409,7 @@ auto HexagonVectorCombine::isSafeToMoveBeforeInBB(const Instruction &In,
   return true;
 }
 
+#ifndef NDEBUG
 auto HexagonVectorCombine::isByteVecTy(Type *Ty) const -> bool {
   if (auto *VecTy = dyn_cast<VectorType>(Ty))
     return VecTy->getElementType() == getByteTy();
@@ -1420,6 +1424,7 @@ auto HexagonVectorCombine::isSectorTy(Type *Ty) const -> bool {
     return Size == static_cast<int>(HST.getVectorLength());
   return Size == 4 || Size == 8;
 }
+#endif
 
 auto HexagonVectorCombine::getElementRange(IRBuilder<> &Builder, Value *Lo,
                                            Value *Hi, int Start,

From b69fe6a85db43df27ebb260716d41a3e1b0d7534 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 6 Jan 2021 15:44:36 +0000
Subject: [PATCH 06/12] [X86] Add icmp ne/eq (srl (ctlz x), log2(bw)) test
 coverage.

Add vector coverage as well (which isn't currently supported).
---
 llvm/test/CodeGen/X86/lzcnt-cmp.ll | 258 +++++++++++++++++++++++++++++
 1 file changed, 258 insertions(+)
 create mode 100644 llvm/test/CodeGen/X86/lzcnt-cmp.ll

diff --git a/llvm/test/CodeGen/X86/lzcnt-cmp.ll b/llvm/test/CodeGen/X86/lzcnt-cmp.ll
new file mode 100644
index 00000000000000..435b09dd5d088b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/lzcnt-cmp.ll
@@ -0,0 +1,258 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+lzcnt | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+lzcnt | FileCheck %s --check-prefix=X64
+
+define i1 @lshr_ctlz_cmpeq_one_i64(i64 %in) {
+; X86-LABEL: lshr_ctlz_cmpeq_one_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: lshr_ctlz_cmpeq_one_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    testq %rdi, %rdi
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 0)
+  %lshr = lshr i64 %ctlz, 6
+  %icmp = icmp eq i64 %lshr, 1
+  ret i1 %icmp
+}
+
+define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) {
+; X86-LABEL: lshr_ctlz_undef_cmpeq_one_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    jne .LBB1_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    addl $32, %eax
+; X86-NEXT:  .LBB1_2:
+; X86-NEXT:    testb $64, %al
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: lshr_ctlz_undef_cmpeq_one_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    lzcntq %rdi, %rax
+; X64-NEXT:    shrq $6, %rax
+; X64-NEXT:    cmpl $1, %eax
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 -1)
+  %lshr = lshr i64 %ctlz, 6
+  %icmp = icmp eq i64 %lshr, 1
+  ret i1 %icmp
+}
+
+define i1 @lshr_ctlz_cmpne_zero_i64(i64 %in) {
+; X86-LABEL: lshr_ctlz_cmpne_zero_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sete %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: lshr_ctlz_cmpne_zero_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    testq %rdi, %rdi
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+  %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 0)
+  %lshr = lshr i64 %ctlz, 6
+  %icmp = icmp ne i64 %lshr, 0
+  ret i1 %icmp
+}
+
+define i1 @lshr_ctlz_undef_cmpne_zero_i64(i64 %in) {
+; X86-LABEL: lshr_ctlz_undef_cmpne_zero_i64:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    jne .LBB3_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    addl $32, %eax
+; X86-NEXT:  .LBB3_2:
+; X86-NEXT:    testb $64, %al
+; X86-NEXT:    setne %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: lshr_ctlz_undef_cmpne_zero_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    lzcntq %rdi, %rax
+; X64-NEXT:    testb $64, %al
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+  %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 -1)
+  %lshr = lshr i64 %ctlz, 6
+  %icmp = icmp ne i64 %lshr, 0
+  ret i1 %icmp
+}
+
+define <2 x i64> @lshr_ctlz_cmpeq_zero_v2i64(<2 x i64> %in) {
+; X86-LABEL: lshr_ctlz_cmpeq_zero_v2i64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl $0, %ecx
+; X86-NEXT:    jne .LBB4_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl $32, %ecx
+; X86-NEXT:  .LBB4_2:
+; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    jne .LBB4_4
+; X86-NEXT:  # %bb.3:
+; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    addl $32, %edx
+; X86-NEXT:  .LBB4_4:
+; X86-NEXT:    andl $-64, %edx
+; X86-NEXT:    cmpl $1, %edx
+; X86-NEXT:    sbbl %edx, %edx
+; X86-NEXT:    andl $-64, %ecx
+; X86-NEXT:    cmpl $1, %ecx
+; X86-NEXT:    sbbl %ecx, %ecx
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %ecx, 8(%eax)
+; X86-NEXT:    movl %edx, 4(%eax)
+; X86-NEXT:    movl %edx, (%eax)
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: lshr_ctlz_cmpeq_zero_v2i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa %xmm0, %xmm1
+; X64-NEXT:    psrlq $1, %xmm1
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm0
+; X64-NEXT:    psrlq $2, %xmm0
+; X64-NEXT:    por %xmm1, %xmm0
+; X64-NEXT:    movdqa %xmm0, %xmm1
+; X64-NEXT:    psrlq $4, %xmm1
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm0
+; X64-NEXT:    psrlq $8, %xmm0
+; X64-NEXT:    por %xmm1, %xmm0
+; X64-NEXT:    movdqa %xmm0, %xmm1
+; X64-NEXT:    psrlq $16, %xmm1
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm0
+; X64-NEXT:    psrlq $32, %xmm0
+; X64-NEXT:    por %xmm1, %xmm0
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    pxor %xmm0, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm0
+; X64-NEXT:    psrlw $1, %xmm0
+; X64-NEXT:    pand {{.*}}(%rip), %xmm0
+; X64-NEXT:    psubb %xmm0, %xmm1
+; X64-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    pand %xmm0, %xmm2
+; X64-NEXT:    psrlw $2, %xmm1
+; X64-NEXT:    pand %xmm0, %xmm1
+; X64-NEXT:    paddb %xmm2, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    psrlw $4, %xmm2
+; X64-NEXT:    paddb %xmm1, %xmm2
+; X64-NEXT:    pand {{.*}}(%rip), %xmm2
+; X64-NEXT:    pxor %xmm0, %xmm0
+; X64-NEXT:    psadbw %xmm0, %xmm2
+; X64-NEXT:    psrlq $6, %xmm2
+; X64-NEXT:    pcmpeqd %xmm0, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; X64-NEXT:    pand %xmm2, %xmm0
+; X64-NEXT:    retq
+  %ctlz = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0)
+  %lshr = lshr <2 x i64> %ctlz, <i64 6, i64 6>
+  %icmp = icmp eq <2 x i64> %lshr, zeroinitializer
+  %sext = sext <2 x i1> %icmp to <2 x i64>
+  ret <2 x i64> %sext
+}
+
+define <2 x i64> @lshr_ctlz_cmpne_zero_v2i64(<2 x i64> %in) {
+; X86-LABEL: lshr_ctlz_cmpne_zero_v2i64:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl $0, %ecx
+; X86-NEXT:    jne .LBB5_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl $32, %ecx
+; X86-NEXT:  .LBB5_2:
+; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT:    jne .LBB5_4
+; X86-NEXT:  # %bb.3:
+; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    addl $32, %edx
+; X86-NEXT:  .LBB5_4:
+; X86-NEXT:    andl $-64, %edx
+; X86-NEXT:    negl %edx
+; X86-NEXT:    sbbl %edx, %edx
+; X86-NEXT:    andl $-64, %ecx
+; X86-NEXT:    negl %ecx
+; X86-NEXT:    sbbl %ecx, %ecx
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %ecx, 8(%eax)
+; X86-NEXT:    movl %edx, 4(%eax)
+; X86-NEXT:    movl %edx, (%eax)
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: lshr_ctlz_cmpne_zero_v2i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa %xmm0, %xmm1
+; X64-NEXT:    psrlq $1, %xmm1
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm0
+; X64-NEXT:    psrlq $2, %xmm0
+; X64-NEXT:    por %xmm1, %xmm0
+; X64-NEXT:    movdqa %xmm0, %xmm1
+; X64-NEXT:    psrlq $4, %xmm1
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm0
+; X64-NEXT:    psrlq $8, %xmm0
+; X64-NEXT:    por %xmm1, %xmm0
+; X64-NEXT:    movdqa %xmm0, %xmm1
+; X64-NEXT:    psrlq $16, %xmm1
+; X64-NEXT:    por %xmm0, %xmm1
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    psrlq $32, %xmm2
+; X64-NEXT:    por %xmm1, %xmm2
+; X64-NEXT:    pcmpeqd %xmm1, %xmm1
+; X64-NEXT:    pxor %xmm1, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    psrlw $1, %xmm0
+; X64-NEXT:    pand {{.*}}(%rip), %xmm0
+; X64-NEXT:    psubb %xmm0, %xmm2
+; X64-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X64-NEXT:    movdqa %xmm2, %xmm3
+; X64-NEXT:    pand %xmm0, %xmm3
+; X64-NEXT:    psrlw $2, %xmm2
+; X64-NEXT:    pand %xmm0, %xmm2
+; X64-NEXT:    paddb %xmm3, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    psrlw $4, %xmm0
+; X64-NEXT:    paddb %xmm2, %xmm0
+; X64-NEXT:    pand {{.*}}(%rip), %xmm0
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    psadbw %xmm2, %xmm0
+; X64-NEXT:    psrlq $6, %xmm0
+; X64-NEXT:    pcmpeqd %xmm2, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,0,3,2]
+; X64-NEXT:    pand %xmm2, %xmm0
+; X64-NEXT:    pxor %xmm1, %xmm0
+; X64-NEXT:    retq
+  %ctlz = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0)
+  %lshr = lshr <2 x i64> %ctlz, <i64 6, i64 6>
+  %icmp = icmp ne <2 x i64> %lshr, zeroinitializer
+  %sext = sext <2 x i1> %icmp to <2 x i64>
+  ret <2 x i64> %sext
+}
+
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)

From 500864f928c272e8ebfd6493cb749083124bfd8b Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 6 Jan 2021 15:50:11 +0000
Subject: [PATCH 07/12] Remove some unused <vector> includes. NFCI.

<vector> (unlike many other c++ headers) is relatively clean, so if the file doesn't use std::vector then it shouldn't need the header.
---
 llvm/include/llvm/Analysis/InlineAdvisor.h           | 6 ++----
 llvm/include/llvm/CodeGen/CodeGenPassBuilder.h       | 1 -
 llvm/include/llvm/ExecutionEngine/JITEventListener.h | 1 -
 3 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 4dbd5786ac7dd5..f051706dca16c5 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -9,13 +9,11 @@
 #ifndef LLVM_INLINEADVISOR_H_
 #define LLVM_INLINEADVISOR_H_
 
-#include <memory>
-#include <unordered_set>
-#include <vector>
-
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/PassManager.h"
+#include <memory>
+#include <unordered_set>
 
 namespace llvm {
 class BasicBlock;
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index b47aaa53eb89ff..893bc6e013f403 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -57,7 +57,6 @@
 #include <string>
 #include <type_traits>
 #include <utility>
-#include <vector>
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/ExecutionEngine/JITEventListener.h b/llvm/include/llvm/ExecutionEngine/JITEventListener.h
index 606b6f7cc12841..4eefd993de2be4 100644
--- a/llvm/include/llvm/ExecutionEngine/JITEventListener.h
+++ b/llvm/include/llvm/ExecutionEngine/JITEventListener.h
@@ -20,7 +20,6 @@
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include <cstdint>
-#include <vector>
 
 namespace llvm {
 

From 350247a93c07906300b79955ff882004a92ae368 Mon Sep 17 00:00:00 2001
From: Nicholas Guy <nicholas.guy@arm.com>
Date: Wed, 2 Dec 2020 15:15:23 +0000
Subject: [PATCH 08/12] [AArch64] Rearrange mul(dup(sext/zext)) to
 mul(sext/zext(dup))

Performing this rearrangement allows for existing patterns
to match cases where the vector may be built after an extend,
instead of before.

Differential Revision: https://reviews.llvm.org/D91255
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 143 ++++++++
 .../AArch64/aarch64-dup-ext-scalable.ll       | 327 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll  | 185 ++++++++++
 3 files changed, 655 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll
 create mode 100644 llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 41dc285a368d17..40435c12ca3b42 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11705,9 +11705,152 @@ static bool IsSVECntIntrinsic(SDValue S) {
   return false;
 }
 
+/// Calculates what the pre-extend type is, based on the extension
+/// operation node provided by \p Extend.
+///
+/// In the case that \p Extend is a SIGN_EXTEND or a ZERO_EXTEND, the
+/// pre-extend type is pulled directly from the operand, while other extend
+/// operations need a bit more inspection to get this information.
+///
+/// \param Extend The SDNode from the DAG that represents the extend operation
+/// \param DAG The SelectionDAG hosting the \p Extend node
+///
+/// \returns The type representing the \p Extend source type, or \p MVT::Other
+/// if no valid type can be determined
+static EVT calculatePreExtendType(SDValue Extend, SelectionDAG &DAG) {
+  switch (Extend.getOpcode()) {
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+    return Extend.getOperand(0).getValueType();
+  case ISD::AssertSext:
+  case ISD::AssertZext:
+  case ISD::SIGN_EXTEND_INREG: {
+    VTSDNode *TypeNode = dyn_cast<VTSDNode>(Extend.getOperand(1));
+    if (!TypeNode)
+      return MVT::Other;
+    return TypeNode->getVT();
+  }
+  case ISD::AND: {
+    ConstantSDNode *Constant =
+        dyn_cast<ConstantSDNode>(Extend.getOperand(1).getNode());
+    if (!Constant)
+      return MVT::Other;
+
+    uint32_t Mask = Constant->getZExtValue();
+
+    if (Mask == UCHAR_MAX)
+      return MVT::i8;
+    else if (Mask == USHRT_MAX)
+      return MVT::i16;
+    else if (Mask == UINT_MAX)
+      return MVT::i32;
+
+    return MVT::Other;
+  }
+  default:
+    return MVT::Other;
+  }
+
+  llvm_unreachable("Code path unhandled in calculatePreExtendType!");
+}
+
+/// Combines a dup(sext/zext) node pattern into sext/zext(dup)
+/// making use of the vector SExt/ZExt rather than the scalar SExt/ZExt
+static SDValue performCommonVectorExtendCombine(SDValue VectorShuffle,
+                                                SelectionDAG &DAG) {
+
+  ShuffleVectorSDNode *ShuffleNode =
+      dyn_cast<ShuffleVectorSDNode>(VectorShuffle.getNode());
+  if (!ShuffleNode)
+    return SDValue();
+
+  // Ensuring the mask is zero before continuing
+  if (!ShuffleNode->isSplat() || ShuffleNode->getSplatIndex() != 0)
+    return SDValue();
+
+  SDValue InsertVectorElt = VectorShuffle.getOperand(0);
+
+  if (InsertVectorElt.getOpcode() != ISD::INSERT_VECTOR_ELT)
+    return SDValue();
+
+  SDValue InsertLane = InsertVectorElt.getOperand(2);
+  ConstantSDNode *Constant = dyn_cast<ConstantSDNode>(InsertLane.getNode());
+  // Ensures the insert is inserting into lane 0
+  if (!Constant || Constant->getZExtValue() != 0)
+    return SDValue();
+
+  SDValue Extend = InsertVectorElt.getOperand(1);
+  unsigned ExtendOpcode = Extend.getOpcode();
+
+  bool IsSExt = ExtendOpcode == ISD::SIGN_EXTEND ||
+                ExtendOpcode == ISD::SIGN_EXTEND_INREG ||
+                ExtendOpcode == ISD::AssertSext;
+  if (!IsSExt && ExtendOpcode != ISD::ZERO_EXTEND &&
+      ExtendOpcode != ISD::AssertZext && ExtendOpcode != ISD::AND)
+    return SDValue();
+
+  EVT TargetType = VectorShuffle.getValueType();
+  EVT PreExtendType = calculatePreExtendType(Extend, DAG);
+
+  if ((TargetType != MVT::v8i16 && TargetType != MVT::v4i32 &&
+       TargetType != MVT::v2i64) ||
+      (PreExtendType == MVT::Other))
+    return SDValue();
+
+  EVT PreExtendVT = TargetType.changeVectorElementType(PreExtendType);
+
+  if (PreExtendVT.getVectorElementCount() != TargetType.getVectorElementCount())
+    return SDValue();
+
+  if (TargetType.getScalarSizeInBits() != PreExtendVT.getScalarSizeInBits() * 2)
+    return SDValue();
+
+  SDLoc DL(VectorShuffle);
+
+  SDValue InsertVectorNode = DAG.getNode(
+      InsertVectorElt.getOpcode(), DL, PreExtendVT, DAG.getUNDEF(PreExtendVT),
+      Extend.getOperand(0), DAG.getConstant(0, DL, MVT::i64));
+
+  std::vector<int> ShuffleMask(TargetType.getVectorElementCount().getValue());
+
+  SDValue VectorShuffleNode =
+      DAG.getVectorShuffle(PreExtendVT, DL, InsertVectorNode,
+                           DAG.getUNDEF(PreExtendVT), ShuffleMask);
+
+  SDValue ExtendNode =
+      DAG.getNode(IsSExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, TargetType,
+                  VectorShuffleNode, DAG.getValueType(TargetType));
+
+  return ExtendNode;
+}
+
+/// Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup))
+/// making use of the vector SExt/ZExt rather than the scalar SExt/ZExt
+static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG) {
+  // If the value type isn't a vector, none of the operands are going to be dups
+  if (!Mul->getValueType(0).isVector())
+    return SDValue();
+
+  SDValue Op0 = performCommonVectorExtendCombine(Mul->getOperand(0), DAG);
+  SDValue Op1 = performCommonVectorExtendCombine(Mul->getOperand(1), DAG);
+
+  // Neither operands have been changed, don't make any further changes
+  if (!Op0 && !Op1)
+    return SDValue();
+
+  SDLoc DL(Mul);
+  return DAG.getNode(Mul->getOpcode(), DL, Mul->getValueType(0),
+                     Op0 ? Op0 : Mul->getOperand(0),
+                     Op1 ? Op1 : Mul->getOperand(1));
+}
+
 static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  const AArch64Subtarget *Subtarget) {
+
+  if (SDValue Ext = performMulVectorExtendCombine(N, DAG))
+    return Ext;
+
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
diff --git a/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll
new file mode 100644
index 00000000000000..082e2db5270536
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll
@@ -0,0 +1,327 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple aarch64-none-linux-gnu -mattr=+sve | FileCheck %s
+
+define <vscale x 2 x i16> @dupsext_v2i8_v2i16(i8 %src, <vscale x 2 x i16> %b) {
+; CHECK-LABEL: dupsext_v2i8_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i16
+    %broadcast.splatinsert = insertelement <vscale x 2 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <vscale x 2 x i16> %broadcast.splatinsert, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nsw <vscale x 2 x i16> %broadcast.splat, %b
+    ret <vscale x 2 x i16> %out
+}
+
+define <vscale x 4 x i16> @dupsext_v4i8_v4i16(i8 %src, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: dupsext_v4i8_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i16
+    %broadcast.splatinsert = insertelement <vscale x 4 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
+    %out = mul nsw <vscale x 4 x i16> %broadcast.splat, %b
+    ret <vscale x 4 x i16> %out
+}
+
+define <vscale x 8 x i16> @dupsext_v8i8_v8i16(i8 %src, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: dupsext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mul z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i16
+    %broadcast.splatinsert = insertelement <vscale x 8 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <vscale x 8 x i16> %broadcast.splatinsert, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+    %out = mul nsw <vscale x 8 x i16> %broadcast.splat, %b
+    ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 2 x i32> @dupsext_v2i8_v2i32(i8 %src, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: dupsext_v2i8_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 2 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 2 x i32> %broadcast.splatinsert, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nsw <vscale x 2 x i32> %broadcast.splat, %b
+    ret <vscale x 2 x i32> %out
+}
+
+define <vscale x 4 x i32> @dupsext_v4i8_v4i32(i8 %src, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: dupsext_v4i8_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 4 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 4 x i32> %broadcast.splatinsert, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+    %out = mul nsw <vscale x 4 x i32> %broadcast.splat, %b
+    ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @dupsext_v2i8_v2i64(i8 %src, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: dupsext_v2i8_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtb x8, w0
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i64
+    %broadcast.splatinsert = insertelement <vscale x 2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <vscale x 2 x i64> %broadcast.splatinsert, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nsw <vscale x 2 x i64> %broadcast.splat, %b
+    ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i32> @dupsext_v2i16_v2i32(i16 %src, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: dupsext_v2i16_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxth w8, w0
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i16 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 2 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 2 x i32> %broadcast.splatinsert, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nsw <vscale x 2 x i32> %broadcast.splat, %b
+    ret <vscale x 2 x i32> %out
+}
+
+define <vscale x 4 x i32> @dupsext_v4i16_v4i32(i16 %src, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: dupsext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxth w8, w0
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i16 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 4 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 4 x i32> %broadcast.splatinsert, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+    %out = mul nsw <vscale x 4 x i32> %broadcast.splat, %b
+    ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @dupsext_v2i16_v2i64(i16 %src, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: dupsext_v2i16_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxth x8, w0
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i16 %src to i64
+    %broadcast.splatinsert = insertelement <vscale x 2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <vscale x 2 x i64> %broadcast.splatinsert, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nsw <vscale x 2 x i64> %broadcast.splat, %b
+    ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i64> @dupsext_v2i32_v2i64(i32 %src, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: dupsext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x8, w0
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i32 %src to i64
+    %broadcast.splatinsert = insertelement <vscale x 2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <vscale x 2 x i64> %broadcast.splatinsert, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nsw <vscale x 2 x i64> %broadcast.splat, %b
+    ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i16> @dupzext_v2i8_v2i16(i8 %src, <vscale x 2 x i16> %b) {
+; CHECK-LABEL: dupzext_v2i8_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i8 %src to i16
+    %broadcast.splatinsert = insertelement <vscale x 2 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <vscale x 2 x i16> %broadcast.splatinsert, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nuw <vscale x 2 x i16> %broadcast.splat, %b
+    ret <vscale x 2 x i16> %out
+}
+
+define <vscale x 4 x i16> @dupzext_v4i8_v4i16(i8 %src, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: dupzext_v4i8_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i8 %src to i16
+    %broadcast.splatinsert = insertelement <vscale x 4 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
+    %out = mul nuw <vscale x 4 x i16> %broadcast.splat, %b
+    ret <vscale x 4 x i16> %out
+}
+
+define <vscale x 8 x i16> @dupzext_v8i8_v8i16(i8 %src, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: dupzext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mul z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i8 %src to i16
+    %broadcast.splatinsert = insertelement <vscale x 8 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <vscale x 8 x i16> %broadcast.splatinsert, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+    %out = mul nuw <vscale x 8 x i16> %broadcast.splat, %b
+    ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 2 x i32> @dupzext_v2i8_v2i32(i8 %src, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: dupzext_v2i8_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i8 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 2 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 2 x i32> %broadcast.splatinsert, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nuw <vscale x 2 x i32> %broadcast.splat, %b
+    ret <vscale x 2 x i32> %out
+}
+
+define <vscale x 4 x i32> @dupzext_v4i8_v4i32(i8 %src, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: dupzext_v4i8_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i8 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 4 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 4 x i32> %broadcast.splatinsert, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+    %out = mul nuw <vscale x 4 x i32> %broadcast.splat, %b
+    ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @dupzext_v2i8_v2i64(i8 %src, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: dupzext_v2i8_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x8, x0, #0xff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i8 %src to i64
+    %broadcast.splatinsert = insertelement <vscale x 2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <vscale x 2 x i64> %broadcast.splatinsert, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nuw <vscale x 2 x i64> %broadcast.splat, %b
+    ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i32> @dupzext_v2i16_v2i32(i16 %src, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: dupzext_v2i16_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i16 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 2 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 2 x i32> %broadcast.splatinsert, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nuw <vscale x 2 x i32> %broadcast.splat, %b
+    ret <vscale x 2 x i32> %out
+}
+
+define <vscale x 4 x i32> @dupzext_v4i16_v4i32(i16 %src, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: dupzext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i16 %src to i32
+    %broadcast.splatinsert = insertelement <vscale x 4 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <vscale x 4 x i32> %broadcast.splatinsert, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+    %out = mul nuw <vscale x 4 x i32> %broadcast.splat, %b
+    ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @dupzext_v2i16_v2i64(i16 %src, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: dupzext_v2i16_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x8, x0, #0xffff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i16 %src to i64
+    %broadcast.splatinsert = insertelement <vscale x 2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <vscale x 2 x i64> %broadcast.splatinsert, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nuw <vscale x 2 x i64> %broadcast.splat, %b
+    ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 2 x i64> @dupzext_v2i32_v2i64(i32 %src, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: dupzext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i32 %src to i64
+    %broadcast.splatinsert = insertelement <vscale x 2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <vscale x 2 x i64> %broadcast.splatinsert, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+    %out = mul nuw <vscale x 2 x i64> %broadcast.splat, %b
+    ret <vscale x 2 x i64> %out
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll b/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
new file mode 100644
index 00000000000000..07ac3c87d14304
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-dup-ext.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple aarch64-none-linux-gnu | FileCheck %s
+
+; Supported combines
+
+define <8 x i16> @dupsext_v8i8_v8i16(i8 %src, <8 x i8> %b) {
+; CHECK-LABEL: dupsext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v1.8b, w0
+; CHECK-NEXT:    smull v0.8h, v1.8b, v0.8b
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i16
+    %ext.b = sext <8 x i8> %b to <8 x i16>
+    %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
+    %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
+    ret <8 x i16> %out
+}
+
+define <8 x i16> @dupzext_v8i8_v8i16(i8 %src, <8 x i8> %b) {
+; CHECK-LABEL: dupzext_v8i8_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v1.8b, w0
+; CHECK-NEXT:    umull v0.8h, v1.8b, v0.8b
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i8 %src to i16
+    %ext.b = zext <8 x i8> %b to <8 x i16>
+    %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer
+    %out = mul nuw <8 x i16> %broadcast.splat, %ext.b
+    ret <8 x i16> %out
+}
+
+define <4 x i32> @dupsext_v4i16_v4i32(i16 %src, <4 x i16> %b) {
+; CHECK-LABEL: dupsext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v1.4h, w0
+; CHECK-NEXT:    smull v0.4s, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i16 %src to i32
+    %ext.b = sext <4 x i16> %b to <4 x i32>
+    %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+    %out = mul nsw <4 x i32> %broadcast.splat, %ext.b
+    ret <4 x i32> %out
+}
+
+define <4 x i32> @dupzext_v4i16_v4i32(i16 %src, <4 x i16> %b) {
+; CHECK-LABEL: dupzext_v4i16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v1.4h, w0
+; CHECK-NEXT:    umull v0.4s, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i16 %src to i32
+    %ext.b = zext <4 x i16> %b to <4 x i32>
+    %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %in, i32 0
+    %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+    %out = mul nuw <4 x i32> %broadcast.splat, %ext.b
+    ret <4 x i32> %out
+}
+
+define <2 x i64> @dupsext_v2i32_v2i64(i32 %src, <2 x i32> %b) {
+; CHECK-LABEL: dupsext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v1.2s, w0
+; CHECK-NEXT:    smull v0.2d, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i32 %src to i64
+    %ext.b = sext <2 x i32> %b to <2 x i64>
+    %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
+    %out = mul nsw <2 x i64> %broadcast.splat, %ext.b
+    ret <2 x i64> %out
+}
+
+define <2 x i64> @dupzext_v2i32_v2i64(i32 %src, <2 x i32> %b) {
+; CHECK-LABEL: dupzext_v2i32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    dup v1.2s, w0
+; CHECK-NEXT:    umull v0.2d, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i32 %src to i64
+    %ext.b = zext <2 x i32> %b to <2 x i64>
+    %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
+    %out = mul nuw <2 x i64> %broadcast.splat, %ext.b
+    ret <2 x i64> %out
+}
+
+; Unsupported combines
+
+define <2 x i16> @dupsext_v2i8_v2i16(i8 %src, <2 x i8> %b) {
+; CHECK-LABEL: dupsext_v2i8_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NEXT:    dup v1.2s, w8
+; CHECK-NEXT:    mul v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i16
+    %ext.b = sext <2 x i8> %b to <2 x i16>
+    %broadcast.splatinsert = insertelement <2 x i16> undef, i16 %in, i16 0
+    %broadcast.splat = shufflevector <2 x i16> %broadcast.splatinsert, <2 x i16> undef, <2 x i32> zeroinitializer
+    %out = mul nsw <2 x i16> %broadcast.splat, %ext.b
+    ret <2 x i16> %out
+}
+
+define <2 x i64> @dupzext_v2i16_v2i64(i16 %src, <2 x i16> %b) {
+; CHECK-LABEL: dupzext_v2i16_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x8, x0, #0xffff
+; CHECK-NEXT:    fmov x10, d0
+; CHECK-NEXT:    mov x9, v0.d[1]
+; CHECK-NEXT:    mul x10, x8, x10
+; CHECK-NEXT:    mul x8, x8, x9
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+entry:
+    %in = zext i16 %src to i64
+    %ext.b = zext <2 x i16> %b to <2 x i64>
+    %broadcast.splatinsert = insertelement <2 x i64> undef, i64 %in, i64 0
+    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> undef, <2 x i32> zeroinitializer
+    %out = mul nuw <2 x i64> %broadcast.splat, %ext.b
+    ret <2 x i64> %out
+}
+
+; dupsext_v4i8_v4i16
+; dupsext_v2i8_v2i32
+; dupsext_v4i8_v4i32
+; dupsext_v2i8_v2i64
+; dupsext_v2i16_v2i32
+; dupsext_v2i16_v2i64
+; dupzext_v2i8_v2i16
+; dupzext_v4i8_v4i16
+; dupzext_v2i8_v2i32
+; dupzext_v4i8_v4i32
+; dupzext_v2i8_v2i64
+; dupzext_v2i16_v2i32
+; dupzext_v2i16_v2i64
+
+; Unsupported states
+
+define <8 x i16> @nonsplat_shuffleinsert(i8 %src, <8 x i8> %b) {
+; CHECK-LABEL: nonsplat_shuffleinsert:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sxtb w8, w0
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    dup v1.8h, w8
+; CHECK-NEXT:    mul v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    ret
+entry:
+    %in = sext i8 %src to i16
+    %ext.b = sext <8 x i8> %b to <8 x i16>
+    %broadcast.splatinsert = insertelement <8 x i16> undef, i16 %in, i16 1
+    %broadcast.splat = shufflevector <8 x i16> %broadcast.splatinsert, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
+    %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
+    ret <8 x i16> %out
+}
+
+define <8 x i16> @missing_insert(<8 x i8> %b) {
+; CHECK-LABEL: missing_insert:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #4
+; CHECK-NEXT:    mul v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    ret
+entry:
+    %ext.b = sext <8 x i8> %b to <8 x i16>
+    %broadcast.splat = shufflevector <8 x i16> %ext.b, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1>
+    %out = mul nsw <8 x i16> %broadcast.splat, %ext.b
+    ret <8 x i16> %out
+}

From 1307e3f6c46cc3a6e6ad9cd46fc67efafcac939e Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 6 Jan 2021 16:13:36 +0000
Subject: [PATCH 09/12] [TargetLowering] Add icmp ne/eq (srl (ctlz x),
 log2(bw)) vector support.

---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |  41 +++--
 llvm/test/CodeGen/X86/lzcnt-cmp.ll            | 171 +++++-------------
 2 files changed, 66 insertions(+), 146 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index d895a53e5a83af..f5abb2c513fbe9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -3486,35 +3486,36 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
     // Optimize some CTPOP cases.
     if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
       return V;
-  }
-
-  // FIXME: Support vectors.
-  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
-    const APInt &C1 = N1C->getAPIntValue();
 
     // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
     // equality comparison, then we're just comparing whether X itself is
     // zero.
     if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
         N0.getOperand(0).getOpcode() == ISD::CTLZ &&
-        N0.getOperand(1).getOpcode() == ISD::Constant) {
-      const APInt &ShAmt = N0.getConstantOperandAPInt(1);
-      if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
-          ShAmt == Log2_32(N0.getValueSizeInBits())) {
-        if ((C1 == 0) == (Cond == ISD::SETEQ)) {
-          // (srl (ctlz x), 5) == 0  -> X != 0
-          // (srl (ctlz x), 5) != 1  -> X != 0
-          Cond = ISD::SETNE;
-        } else {
-          // (srl (ctlz x), 5) != 0  -> X == 0
-          // (srl (ctlz x), 5) == 1  -> X == 0
-          Cond = ISD::SETEQ;
+        isPowerOf2_32(N0.getScalarValueSizeInBits())) {
+      if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
+        if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
+            ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
+          if ((C1 == 0) == (Cond == ISD::SETEQ)) {
+            // (srl (ctlz x), 5) == 0  -> X != 0
+            // (srl (ctlz x), 5) != 1  -> X != 0
+            Cond = ISD::SETNE;
+          } else {
+            // (srl (ctlz x), 5) != 0  -> X == 0
+            // (srl (ctlz x), 5) == 1  -> X == 0
+            Cond = ISD::SETEQ;
+          }
+          SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
+          return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
+                              Cond);
         }
-        SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
-        return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
-                            Zero, Cond);
       }
     }
+  }
+
+  // FIXME: Support vectors.
+  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
+    const APInt &C1 = N1C->getAPIntValue();
 
     // (zext x) == C --> x == (trunc C)
     // (sext x) == C --> x == (trunc C)
diff --git a/llvm/test/CodeGen/X86/lzcnt-cmp.ll b/llvm/test/CodeGen/X86/lzcnt-cmp.ll
index 435b09dd5d088b..3823524f552a25 100644
--- a/llvm/test/CodeGen/X86/lzcnt-cmp.ll
+++ b/llvm/test/CodeGen/X86/lzcnt-cmp.ll
@@ -96,75 +96,36 @@ define i1 @lshr_ctlz_undef_cmpne_zero_i64(i64 %in) {
 define <2 x i64> @lshr_ctlz_cmpeq_zero_v2i64(<2 x i64> %in) {
 ; X86-LABEL: lshr_ctlz_cmpeq_zero_v2i64:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %esi, -8
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    setne %cl
+; X86-NEXT:    negl %ecx
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl $0, %ecx
-; X86-NEXT:    jne .LBB4_2
-; X86-NEXT:  # %bb.1:
-; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    addl $32, %ecx
-; X86-NEXT:  .LBB4_2:
-; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    jne .LBB4_4
-; X86-NEXT:  # %bb.3:
-; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    addl $32, %edx
-; X86-NEXT:  .LBB4_4:
-; X86-NEXT:    andl $-64, %edx
-; X86-NEXT:    cmpl $1, %edx
-; X86-NEXT:    sbbl %edx, %edx
-; X86-NEXT:    andl $-64, %ecx
-; X86-NEXT:    cmpl $1, %ecx
-; X86-NEXT:    sbbl %ecx, %ecx
-; X86-NEXT:    movl %ecx, 12(%eax)
-; X86-NEXT:    movl %ecx, 8(%eax)
-; X86-NEXT:    movl %edx, 4(%eax)
-; X86-NEXT:    movl %edx, (%eax)
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    setne %dl
+; X86-NEXT:    negl %edx
+; X86-NEXT:    movl %edx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: lshr_ctlz_cmpeq_zero_v2i64:
 ; X64:       # %bb.0:
-; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psrlq $1, %xmm1
-; X64-NEXT:    por %xmm0, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm0
-; X64-NEXT:    psrlq $2, %xmm0
-; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psrlq $4, %xmm1
-; X64-NEXT:    por %xmm0, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm0
-; X64-NEXT:    psrlq $8, %xmm0
-; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psrlq $16, %xmm1
-; X64-NEXT:    por %xmm0, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm0
-; X64-NEXT:    psrlq $32, %xmm0
-; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-NEXT:    pxor %xmm0, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm0
-; X64-NEXT:    psrlw $1, %xmm0
-; X64-NEXT:    pand {{.*}}(%rip), %xmm0
-; X64-NEXT:    psubb %xmm0, %xmm1
-; X64-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-NEXT:    movdqa %xmm1, %xmm2
-; X64-NEXT:    pand %xmm0, %xmm2
-; X64-NEXT:    psrlw $2, %xmm1
-; X64-NEXT:    pand %xmm0, %xmm1
-; X64-NEXT:    paddb %xmm2, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm2
-; X64-NEXT:    psrlw $4, %xmm2
-; X64-NEXT:    paddb %xmm1, %xmm2
-; X64-NEXT:    pand {{.*}}(%rip), %xmm2
-; X64-NEXT:    pxor %xmm0, %xmm0
-; X64-NEXT:    psadbw %xmm0, %xmm2
-; X64-NEXT:    psrlq $6, %xmm2
-; X64-NEXT:    pcmpeqd %xmm0, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; X64-NEXT:    pand %xmm2, %xmm0
+; X64-NEXT:    pxor %xmm1, %xmm1
+; X64-NEXT:    pcmpeqd %xmm0, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,0,3,2]
+; X64-NEXT:    pand %xmm1, %xmm2
+; X64-NEXT:    pcmpeqd %xmm0, %xmm0
+; X64-NEXT:    pxor %xmm2, %xmm0
 ; X64-NEXT:    retq
   %ctlz = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0)
   %lshr = lshr <2 x i64> %ctlz, <i64 6, i64 6>
@@ -176,76 +137,34 @@ define <2 x i64> @lshr_ctlz_cmpeq_zero_v2i64(<2 x i64> %in) {
 define <2 x i64> @lshr_ctlz_cmpne_zero_v2i64(<2 x i64> %in) {
 ; X86-LABEL: lshr_ctlz_cmpne_zero_v2i64:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %esi, -8
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    sete %cl
+; X86-NEXT:    negl %ecx
 ; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl $0, %ecx
-; X86-NEXT:    jne .LBB5_2
-; X86-NEXT:  # %bb.1:
-; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    addl $32, %ecx
-; X86-NEXT:  .LBB5_2:
-; X86-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    jne .LBB5_4
-; X86-NEXT:  # %bb.3:
-; X86-NEXT:    lzcntl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    addl $32, %edx
-; X86-NEXT:  .LBB5_4:
-; X86-NEXT:    andl $-64, %edx
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    sete %dl
 ; X86-NEXT:    negl %edx
-; X86-NEXT:    sbbl %edx, %edx
-; X86-NEXT:    andl $-64, %ecx
-; X86-NEXT:    negl %ecx
-; X86-NEXT:    sbbl %ecx, %ecx
-; X86-NEXT:    movl %ecx, 12(%eax)
-; X86-NEXT:    movl %ecx, 8(%eax)
-; X86-NEXT:    movl %edx, 4(%eax)
-; X86-NEXT:    movl %edx, (%eax)
+; X86-NEXT:    movl %edx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl $4
 ;
 ; X64-LABEL: lshr_ctlz_cmpne_zero_v2i64:
 ; X64:       # %bb.0:
-; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psrlq $1, %xmm1
-; X64-NEXT:    por %xmm0, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm0
-; X64-NEXT:    psrlq $2, %xmm0
-; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psrlq $4, %xmm1
-; X64-NEXT:    por %xmm0, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm0
-; X64-NEXT:    psrlq $8, %xmm0
-; X64-NEXT:    por %xmm1, %xmm0
-; X64-NEXT:    movdqa %xmm0, %xmm1
-; X64-NEXT:    psrlq $16, %xmm1
-; X64-NEXT:    por %xmm0, %xmm1
-; X64-NEXT:    movdqa %xmm1, %xmm2
-; X64-NEXT:    psrlq $32, %xmm2
-; X64-NEXT:    por %xmm1, %xmm2
-; X64-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-NEXT:    pxor %xmm1, %xmm2
-; X64-NEXT:    movdqa %xmm2, %xmm0
-; X64-NEXT:    psrlw $1, %xmm0
-; X64-NEXT:    pand {{.*}}(%rip), %xmm0
-; X64-NEXT:    psubb %xmm0, %xmm2
-; X64-NEXT:    movdqa {{.*#+}} xmm0 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-NEXT:    movdqa %xmm2, %xmm3
-; X64-NEXT:    pand %xmm0, %xmm3
-; X64-NEXT:    psrlw $2, %xmm2
-; X64-NEXT:    pand %xmm0, %xmm2
-; X64-NEXT:    paddb %xmm3, %xmm2
-; X64-NEXT:    movdqa %xmm2, %xmm0
-; X64-NEXT:    psrlw $4, %xmm0
-; X64-NEXT:    paddb %xmm2, %xmm0
-; X64-NEXT:    pand {{.*}}(%rip), %xmm0
-; X64-NEXT:    pxor %xmm2, %xmm2
-; X64-NEXT:    psadbw %xmm2, %xmm0
-; X64-NEXT:    psrlq $6, %xmm0
-; X64-NEXT:    pcmpeqd %xmm2, %xmm0
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,0,3,2]
-; X64-NEXT:    pand %xmm2, %xmm0
-; X64-NEXT:    pxor %xmm1, %xmm0
+; X64-NEXT:    pxor %xmm1, %xmm1
+; X64-NEXT:    pcmpeqd %xmm0, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
+; X64-NEXT:    pand %xmm1, %xmm0
 ; X64-NEXT:    retq
   %ctlz = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %in, i1 0)
   %lshr = lshr <2 x i64> %ctlz, <i64 6, i64 6>

From 7809fa20400000fd40b4a4b56696c7fbcd0f0fa9 Mon Sep 17 00:00:00 2001
From: Faris Rehman <faris.rehman@arm.com>
Date: Wed, 6 Jan 2021 15:42:24 +0000
Subject: [PATCH 10/12] [flang][driver] Add support for `-D`, `-U`

Add support for options -D and -U in the new Flang driver.

Summary of changes:
  - Create PreprocessorOptions, to be used by the driver then translated
    into Fortran::parser::Options
  - Create CompilerInvocation::setFortranOpts to pass preprocessor
    options into the parser options
  - Add a dedicated method, Flang::AddPreprocessingOptions, to extract
    preprocessing options from the driver arguments into the preprocessor
    command arguments

Macros specified like -DName will default to definition 1.

When defining macros, the new driver will drop anything after an
end-of-line character. This is consistent with gfortran and clang, but
different to what currently f18 does. However, flang (which is a bash
wrapper for f18), also drops everything after an end-of-line character.
So gfortran-like behaviour felt like the natural choice. Test is added
to demonstrate this behaviour.

Reviewed By: awarzynski

Differential Revision: https://reviews.llvm.org/D93401
---
 clang/include/clang/Driver/Options.td         |  4 +-
 clang/lib/Driver/ToolChains/Flang.cpp         | 14 +++-
 clang/lib/Driver/ToolChains/Flang.h           |  9 +++
 .../include/flang/Frontend/CompilerInstance.h |  8 +++
 .../flang/Frontend/CompilerInvocation.h       | 13 ++++
 .../flang/Frontend/PreprocessorOptions.h      | 42 +++++++++++
 flang/lib/Frontend/CompilerInstance.cpp       |  2 +
 flang/lib/Frontend/CompilerInvocation.cpp     | 69 ++++++++++++++++++-
 .../test/Flang-Driver/driver-help-hidden.f90  |  2 +
 flang/test/Flang-Driver/driver-help.f90       | 12 ++--
 flang/test/Flang-Driver/macro_def_undef.f90   | 38 ++++++++++
 flang/test/Flang-Driver/macro_multiline.f90   | 22 ++++++
 12 files changed, 226 insertions(+), 9 deletions(-)
 create mode 100644 flang/include/flang/Frontend/PreprocessorOptions.h
 create mode 100644 flang/test/Flang-Driver/macro_def_undef.f90
 create mode 100644 flang/test/Flang-Driver/macro_multiline.f90

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 3c2a9f307c659c..428c14a7d9bbed 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -631,7 +631,7 @@ def C : Flag<["-"], "C">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
     HelpText<"Include comments in preprocessed output">,
     MarshallingInfoFlag<"PreprocessorOutputOpts.ShowComments">;
 def D : JoinedOrSeparate<["-"], "D">, Group<Preprocessor_Group>,
-    Flags<[CC1Option]>, MetaVarName<"<macro>=<value>">,
+    Flags<[CC1Option, FlangOption, FC1Option]>, MetaVarName<"<macro>=<value>">,
     HelpText<"Define <macro> to <value> (or 1 if <value> omitted)">;
 def E : Flag<["-"], "E">, Flags<[NoXarchOption,CC1Option, FlangOption, FC1Option]>, Group<Action_Group>,
     HelpText<"Only run the preprocessor">;
@@ -730,7 +730,7 @@ def Ttext : JoinedOrSeparate<["-"], "Ttext">, Group<T_Group>,
 def T : JoinedOrSeparate<["-"], "T">, Group<T_Group>,
   MetaVarName<"<script>">, HelpText<"Specify <script> as linker script">;
 def U : JoinedOrSeparate<["-"], "U">, Group<Preprocessor_Group>,
-  Flags<[CC1Option]>, MetaVarName<"<macro>">, HelpText<"Undefine macro <macro>">;
+  Flags<[CC1Option, FlangOption, FC1Option]>, MetaVarName<"<macro>">, HelpText<"Undefine macro <macro>">;
 def V : JoinedOrSeparate<["-"], "V">, Flags<[NoXarchOption, Unsupported]>;
 def Wa_COMMA : CommaJoined<["-"], "Wa,">,
   HelpText<"Pass the comma separated arguments in <arg> to the assembler">,
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index d7dee9594e4509..69f0841bb0c596 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -19,6 +19,11 @@ using namespace clang::driver::tools;
 using namespace clang;
 using namespace llvm::opt;
 
+void Flang::AddPreprocessingOptions(const ArgList &Args,
+                                    ArgStringList &CmdArgs) const {
+  Args.AddAllArgs(CmdArgs, {options::OPT_D, options::OPT_U});
+}
+
 void Flang::ConstructJob(Compilation &C, const JobAction &JA,
                          const InputInfo &Output, const InputInfoList &Inputs,
                          const ArgList &Args, const char *LinkingOutput) const {
@@ -63,6 +68,14 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
     assert(false && "Unexpected action class for Flang tool.");
   }
 
+  const InputInfo &Input = Inputs[0];
+  types::ID InputType = Input.getType();
+
+  // Add preprocessing options like -I, -D, etc. if we are using the
+  // preprocessor (i.e. skip when dealing with e.g. binary files).
+  if (types::getPreprocessedType(InputType) != types::TY_INVALID)
+    AddPreprocessingOptions(Args, CmdArgs);
+
   if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
@@ -70,7 +83,6 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
     assert(Output.isNothing() && "Invalid output.");
   }
 
-  const InputInfo &Input = Inputs[0];
   assert(Input.isFilename() && "Invalid input.");
   CmdArgs.push_back(Input.getFilename());
 
diff --git a/clang/lib/Driver/ToolChains/Flang.h b/clang/lib/Driver/ToolChains/Flang.h
index 19e3a8c28f7e42..83b79505e0a90f 100644
--- a/clang/lib/Driver/ToolChains/Flang.h
+++ b/clang/lib/Driver/ToolChains/Flang.h
@@ -23,6 +23,15 @@ namespace tools {
 
 /// Flang compiler tool.
 class LLVM_LIBRARY_VISIBILITY Flang : public Tool {
+private:
+  /// Extract preprocessing options from the driver arguments and add them to
+  /// the preprocessor command arguments.
+  ///
+  /// \param [in] Args The list of input driver arguments
+  /// \param [out] CmdArgs The list of output command arguments
+  void AddPreprocessingOptions(const llvm::opt::ArgList &Args,
+                               llvm::opt::ArgStringList &CmdArgs) const;
+
 public:
   Flang(const ToolChain &TC);
   ~Flang() override;
diff --git a/flang/include/flang/Frontend/CompilerInstance.h b/flang/include/flang/Frontend/CompilerInstance.h
index b00b5cd4479a68..79a05c0ddbbeb4 100644
--- a/flang/include/flang/Frontend/CompilerInstance.h
+++ b/flang/include/flang/Frontend/CompilerInstance.h
@@ -10,6 +10,7 @@
 
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/FrontendAction.h"
+#include "flang/Frontend/PreprocessorOptions.h"
 #include "flang/Parser/parsing.h"
 #include "flang/Parser/provenance.h"
 #include "flang/Semantics/semantics.h"
@@ -135,6 +136,13 @@ class CompilerInstance {
     return invocation_->frontendOpts();
   }
 
+  PreprocessorOptions &preprocessorOpts() {
+    return invocation_->preprocessorOpts();
+  }
+  const PreprocessorOptions &preprocessorOpts() const {
+    return invocation_->preprocessorOpts();
+  }
+
   /// }
   /// @name Diagnostics Engine
   /// {
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index 12915c7c1d9cef..5136de54e06b0a 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -9,10 +9,12 @@
 #define LLVM_FLANG_FRONTEND_COMPILERINVOCATION_H
 
 #include "flang/Frontend/FrontendOptions.h"
+#include "flang/Frontend/PreprocessorOptions.h"
 #include "flang/Parser/parsing.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "llvm/Option/ArgList.h"
+#include <memory>
 
 namespace Fortran::frontend {
 
@@ -27,6 +29,8 @@ class CompilerInvocationBase {
 public:
   /// Options controlling the diagnostic engine.
   llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> diagnosticOpts_;
+  /// Options for the preprocessor.
+  std::shared_ptr<Fortran::frontend::PreprocessorOptions> preprocessorOpts_;
 
   CompilerInvocationBase();
   CompilerInvocationBase(const CompilerInvocationBase &x);
@@ -38,6 +42,11 @@ class CompilerInvocationBase {
   const clang::DiagnosticOptions &GetDiagnosticOpts() const {
     return *diagnosticOpts_.get();
   }
+
+  PreprocessorOptions &preprocessorOpts() { return *preprocessorOpts_; }
+  const PreprocessorOptions &preprocessorOpts() const {
+    return *preprocessorOpts_;
+  }
 };
 
 class CompilerInvocation : public CompilerInvocationBase {
@@ -74,6 +83,10 @@ class CompilerInvocation : public CompilerInvocationBase {
   // need to extend frontendOpts_ first. Next, we need to add the corresponding
   // compiler driver options in libclangDriver.
   void SetDefaultFortranOpts();
+
+  /// Set the Fortran options to user-specified values.
+  /// These values are found in the preprocessor options.
+  void setFortranOpts();
 };
 
 } // end namespace Fortran::frontend
diff --git a/flang/include/flang/Frontend/PreprocessorOptions.h b/flang/include/flang/Frontend/PreprocessorOptions.h
new file mode 100644
index 00000000000000..d182969eb78b26
--- /dev/null
+++ b/flang/include/flang/Frontend/PreprocessorOptions.h
@@ -0,0 +1,42 @@
+//===- PreprocessorOptions.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declaration of the PreprocessorOptions class, which
+/// is the class for all preprocessor options.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FLANG_PREPROCESSOROPTIONS_H
+#define LLVM_FLANG_PREPROCESSOROPTIONS_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace Fortran::frontend {
+
+/// This class is used for passing the various options used
+/// in preprocessor initialization to the parser options.
+class PreprocessorOptions {
+public:
+  std::vector<std::pair<std::string, /*isUndef*/ bool>> macros;
+
+public:
+  PreprocessorOptions() {}
+
+  void addMacroDef(llvm::StringRef name) {
+    macros.emplace_back(std::string(name), false);
+  }
+
+  void addMacroUndef(llvm::StringRef name) {
+    macros.emplace_back(std::string(name), true);
+  }
+};
+
+} // namespace Fortran::frontend
+
+#endif // LLVM_FLANG_PREPROCESSOROPTIONS_H
\ No newline at end of file
diff --git a/flang/lib/Frontend/CompilerInstance.cpp b/flang/lib/Frontend/CompilerInstance.cpp
index f473bcd1954692..589f8204c60f32 100644
--- a/flang/lib/Frontend/CompilerInstance.cpp
+++ b/flang/lib/Frontend/CompilerInstance.cpp
@@ -142,6 +142,8 @@ bool CompilerInstance::ExecuteAction(FrontendAction &act) {
   // TODO: Instead of defaults we should be setting these options based on the
   // user input.
   this->invocation().SetDefaultFortranOpts();
+  // Set the fortran options to user-based input.
+  this->invocation().setFortranOpts();
 
   // Connect Input to a CompileInstance
   for (const FrontendInputFile &fif : frontendOpts().inputs_) {
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index db3ae74128cde1..df7fe44feb9cff 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Frontend/CompilerInvocation.h"
+#include "flang/Frontend/PreprocessorOptions.h"
 #include "clang/Basic/AllDiagnostics.h"
 #include "clang/Basic/DiagnosticDriver.h"
 #include "clang/Basic/DiagnosticOptions.h"
@@ -26,10 +27,12 @@ using namespace Fortran::frontend;
 // Initialization.
 //===----------------------------------------------------------------------===//
 CompilerInvocationBase::CompilerInvocationBase()
-    : diagnosticOpts_(new clang::DiagnosticOptions()) {}
+    : diagnosticOpts_(new clang::DiagnosticOptions()),
+      preprocessorOpts_(new PreprocessorOptions()) {}
 
 CompilerInvocationBase::CompilerInvocationBase(const CompilerInvocationBase &x)
-    : diagnosticOpts_(new clang::DiagnosticOptions(x.GetDiagnosticOpts())) {}
+    : diagnosticOpts_(new clang::DiagnosticOptions(x.GetDiagnosticOpts())),
+      preprocessorOpts_(new PreprocessorOptions(x.preprocessorOpts())) {}
 
 CompilerInvocationBase::~CompilerInvocationBase() = default;
 
@@ -155,6 +158,24 @@ static InputKind ParseFrontendArgs(FrontendOptions &opts,
   return dashX;
 }
 
+/// Parses all preprocessor input arguments and populates the preprocessor
+/// options accordingly.
+///
+/// \param [in] opts The preprocessor options instance
+/// \param [out] args The list of input arguments
+static void parsePreprocessorArgs(
+    Fortran::frontend::PreprocessorOptions &opts, llvm::opt::ArgList &args) {
+  // Add macros from the command line.
+  for (const auto *currentArg : args.filtered(
+           clang::driver::options::OPT_D, clang::driver::options::OPT_U)) {
+    if (currentArg->getOption().matches(clang::driver::options::OPT_D)) {
+      opts.addMacroDef(currentArg->getValue());
+    } else {
+      opts.addMacroUndef(currentArg->getValue());
+    }
+  }
+}
+
 bool CompilerInvocation::CreateFromArgs(CompilerInvocation &res,
     llvm::ArrayRef<const char *> commandLineArgs,
     clang::DiagnosticsEngine &diags) {
@@ -183,10 +204,47 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &res,
 
   // Parse the frontend args
   ParseFrontendArgs(res.frontendOpts(), args, diags);
+  // Parse the preprocessor args
+  parsePreprocessorArgs(res.preprocessorOpts(), args);
 
   return success;
 }
 
+/// Collect the macro definitions provided by the given preprocessor
+/// options into the parser options.
+///
+/// \param [in] ppOpts The preprocessor options
+/// \param [out] opts The fortran options
+static void collectMacroDefinitions(
+    const PreprocessorOptions &ppOpts, Fortran::parser::Options &opts) {
+  for (unsigned i = 0, n = ppOpts.macros.size(); i != n; ++i) {
+    llvm::StringRef macro = ppOpts.macros[i].first;
+    bool isUndef = ppOpts.macros[i].second;
+
+    std::pair<llvm::StringRef, llvm::StringRef> macroPair = macro.split('=');
+    llvm::StringRef macroName = macroPair.first;
+    llvm::StringRef macroBody = macroPair.second;
+
+    // For an #undef'd macro, we only care about the name.
+    if (isUndef) {
+      opts.predefinitions.emplace_back(
+          macroName.str(), std::optional<std::string>{});
+      continue;
+    }
+
+    // For a #define'd macro, figure out the actual definition.
+    if (macroName.size() == macro.size())
+      macroBody = "1";
+    else {
+      // Note: GCC drops anything following an end-of-line character.
+      llvm::StringRef::size_type End = macroBody.find_first_of("\n\r");
+      macroBody = macroBody.substr(0, End);
+    }
+    opts.predefinitions.emplace_back(
+        macroName, std::optional<std::string>(macroBody.str()));
+  }
+}
+
 void CompilerInvocation::SetDefaultFortranOpts() {
   auto &fortranOptions = fortranOpts();
 
@@ -195,3 +253,10 @@ void CompilerInvocation::SetDefaultFortranOpts() {
   fortranOptions.searchDirectories = searchDirectories;
   fortranOptions.isFixedForm = false;
 }
+
+void CompilerInvocation::setFortranOpts() {
+  auto &fortranOptions = fortranOpts();
+  const auto &preprocessorOptions = preprocessorOpts();
+
+  collectMacroDefinitions(preprocessorOptions, fortranOptions);
+}
diff --git a/flang/test/Flang-Driver/driver-help-hidden.f90 b/flang/test/Flang-Driver/driver-help-hidden.f90
index 3143cfd3d7152e..2b0e3162eb6f78 100644
--- a/flang/test/Flang-Driver/driver-help-hidden.f90
+++ b/flang/test/Flang-Driver/driver-help-hidden.f90
@@ -19,12 +19,14 @@
 ! CHECK-EMPTY:
 ! CHECK-NEXT:OPTIONS:
 ! CHECK-NEXT: -###      Print (but do not run) the commands to run for this compilation
+! CHECK-NEXT: -D <macro>=<value>     Define <macro> to <value> (or 1 if <value> omitted)
 ! CHECK-NEXT: -E        Only run the preprocessor
 ! CHECK-NEXT: -fcolor-diagnostics    Enable colors in diagnostics
 ! CHECK-NEXT: -fno-color-diagnostics Disable colors in diagnostics
 ! CHECK-NEXT: -help     Display available options
 ! CHECK-NEXT: -o <file> Write output to <file>
 ! CHECK-NEXT: -test-io  Run the InputOuputTest action. Use for development and testing only.
+! CHECK-NEXT: -U <macro>             Undefine macro <macro>
 ! CHECK-NEXT: --version Print version information
 
 !-------------------------------------------------------------
diff --git a/flang/test/Flang-Driver/driver-help.f90 b/flang/test/Flang-Driver/driver-help.f90
index 58fa8fc79aca68..69319253034ced 100644
--- a/flang/test/Flang-Driver/driver-help.f90
+++ b/flang/test/Flang-Driver/driver-help.f90
@@ -19,11 +19,13 @@
 ! HELP-EMPTY:
 ! HELP-NEXT:OPTIONS:
 ! HELP-NEXT: -###                   Print (but do not run) the commands to run for this compilation
+! HELP-NEXT: -D <macro>=<value>     Define <macro> to <value> (or 1 if <value> omitted)
 ! HELP-NEXT: -E                     Only run the preprocessor
 ! HELP-NEXT: -fcolor-diagnostics    Enable colors in diagnostics
 ! HELP-NEXT: -fno-color-diagnostics Disable colors in diagnostics
 ! HELP-NEXT: -help                  Display available options
 ! HELP-NEXT: -o <file>              Write output to <file>
+! HELP-NEXT: -U <macro>             Undefine macro <macro>
 ! HELP-NEXT: --version              Print version information
 
 !-------------------------------------------------------------
@@ -32,10 +34,12 @@
 ! HELP-FC1:USAGE: flang-new
 ! HELP-FC1-EMPTY:
 ! HELP-FC1-NEXT:OPTIONS:
-! HELP-FC1-NEXT: -E        Only run the preprocessor
-! HELP-FC1-NEXT: -help     Display available options
-! HELP-FC1-NEXT: -o <file> Write output to <file>
-! HELP-FC1-NEXT: --version Print version information
+! HELP-FC1-NEXT: -D <macro>=<value>     Define <macro> to <value> (or 1 if <value> omitted)
+! HELP-FC1-NEXT: -E                     Only run the preprocessor
+! HELP-FC1-NEXT: -help                  Display available options
+! HELP-FC1-NEXT: -o <file>              Write output to <file>
+! HELP-FC1-NEXT: -U <macro>             Undefine macro <macro>
+! HELP-FC1-NEXT: --version              Print version information
 
 !---------------
 ! EXPECTED ERROR
diff --git a/flang/test/Flang-Driver/macro_def_undef.f90 b/flang/test/Flang-Driver/macro_def_undef.f90
new file mode 100644
index 00000000000000..694f7676eeb115
--- /dev/null
+++ b/flang/test/Flang-Driver/macro_def_undef.f90
@@ -0,0 +1,38 @@
+! Ensure arguments -D and -U work as expected.
+
+! REQUIRES: new-flang-driver
+
+!--------------------------
+! FLANG DRIVER (flang-new)
+!--------------------------
+! RUN: %flang-new -E %s  2>&1 | FileCheck %s --check-prefix=UNDEFINED
+! RUN: %flang-new -E -DX=A %s  2>&1 | FileCheck %s --check-prefix=DEFINED
+! RUN: %flang-new -E -DX=A -UX %s  2>&1 | FileCheck %s --check-prefix=UNDEFINED
+
+!-----------------------------------------
+! FRONTEND FLANG DRIVER (flang-new -fc1)
+!-----------------------------------------
+! RUN: %flang-new -fc1 -E %s  2>&1 | FileCheck %s --check-prefix=UNDEFINED
+! RUN: %flang-new -fc1 -E -DX=A %s  2>&1 | FileCheck %s --check-prefix=DEFINED
+! RUN: %flang-new -fc1 -E -DX -UX %s  2>&1 | FileCheck %s --check-prefix=UNDEFINED
+
+!--------------------------------------------
+! EXPECTED OUTPUT FOR AN UNDEFINED MACRO
+!--------------------------------------------
+! UNDEFINED:program b
+! UNDEFINED-NOT:program x
+! UNDEFINED-NEXT:end
+
+!--------------------------------------------
+! EXPECTED OUTPUT FOR MACRO 'X' DEFINED AS A
+!--------------------------------------------
+! DEFINED:program a
+! DEFINED-NOT:program b
+! DEFINED-NEXT:end
+
+#ifdef X
+program X
+#else
+program B
+#endif
+end
\ No newline at end of file
diff --git a/flang/test/Flang-Driver/macro_multiline.f90 b/flang/test/Flang-Driver/macro_multiline.f90
new file mode 100644
index 00000000000000..c02996eed5bf5d
--- /dev/null
+++ b/flang/test/Flang-Driver/macro_multiline.f90
@@ -0,0 +1,22 @@
+! Ensure the end-of-line character and anything that follows after in a macro definition (-D) is ignored.
+
+! REQUIRES: new-flang-driver
+
+!--------------------------
+! FLANG DRIVER (flang-new)
+!--------------------------
+! RUN: printf -- "-DX=A\\\\\nTHIS_SHOULD_NOT_EXIST_IN_THE_OUTPUT\n" | xargs %flang-new -E %s  2>&1 | FileCheck --strict-whitespace --match-full-lines %s
+
+!-----------------------------------------
+! FRONTEND FLANG DRIVER (flang-new -fc1)
+!-----------------------------------------
+! RUN: printf -- "-DX=A\\\\\nTHIS_SHOULD_NOT_EXIST_IN_THE_OUTPUT\n" | xargs %flang-new -fc1 -E %s  2>&1 | FileCheck --strict-whitespace --match-full-lines %s
+
+!-------------------------------
+! EXPECTED OUTPUT FOR MACRO 'X'
+!-------------------------------
+! CHECK:start a end
+! CHECK-NOT:THIS_SHOULD_NOT_EXIST_IN_THE_OUTPUT
+! CHECK-NOT:this_should_not_exist_in_the_output
+
+START X END
\ No newline at end of file

From 136f49891953ce232be2f4a8bc98e83bb2cd6462 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 29 Jul 2020 22:50:02 -0400
Subject: [PATCH 11/12] AMDGPU: Explicitly use SelectionDAG in legacy intrinsic
 tests

GlobalISel will probably not support the legacy buffer intrinsics, so
don't fail when the default is switched.
---
 .../CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll   | 6 +++---
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll | 4 ++--
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll    | 6 +++---
 .../test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll | 4 ++--
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll        | 4 ++--
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll   | 6 +++---
 .../CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll     | 2 +-
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll       | 4 ++--
 8 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll
index aadd9a448a1b37..d5bdb62354cac6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.d16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,UNPACKED %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,PACKED,GFX81 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,PACKED,GFX9 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,UNPACKED %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,PACKED,GFX81 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,PACKED,GFX9 %s
 
 ; GCN-LABEL: {{^}}buffer_store_format_d16_x:
 ; GCN: s_load_dword s[[LO:[0-9]+]]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
index 78c5281ed73e16..1a13e0bdd8020e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
@@ -1,5 +1,5 @@
-;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=VERDE %s
-;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
+;RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=VERDE %s
+;RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
 
 ;CHECK-LABEL: {{^}}buffer_store:
 ;CHECK-NOT: s_waitcnt
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
index 2839f92d2aae11..ef6b1fb883eca6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.d16.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=UNPACKED %s
-; RUN: llc < %s -march=amdgcn -mcpu=gfx810 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
-; RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
+; RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=UNPACKED %s
+; RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=gfx810 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
+; RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED %s
 
 ; GCN-LABEL: {{^}}tbuffer_load_d16_x:
 ; GCN: tbuffer_load_format_d16_x v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], 0 format:[BUF_DATA_FORMAT_10_11_11,BUF_NUM_FORMAT_SNORM]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
index b04f3c09729a0d..07a7a69a96c095 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.dwordx3.ll
@@ -1,5 +1,5 @@
-;RUN: llc < %s -march=amdgcn -mcpu=gfx600 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,SI
-;RUN: llc < %s -march=amdgcn -mcpu=gfx700 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,GCNX3
+; RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=gfx600 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,SI
+; RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=gfx700 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,GCNX3
 
 ; GCN-LABEL: {{^}}tbuffer_raw_load_immoffs_x3:
 ; SI: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT] offset:42
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
index aca9e0aaf9a26e..f874ce02a2a746 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.load.ll
@@ -1,5 +1,5 @@
-;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=GCN %s
-;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+;RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+;RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}tbuffer_load:
 ; GCN: tbuffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 format:[BUF_DATA_FORMAT_32_32_32_32,BUF_NUM_FORMAT_UINT]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
index a940df3540cfe2..3fe9dadd2a5702 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.d16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=UNPACKED %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED -check-prefix=GFX81 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED -check-prefix=GFX9 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=UNPACKED %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED -check-prefix=GFX81 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=PACKED -check-prefix=GFX9 %s
 
 
 ; GCN-LABEL: {{^}}tbuffer_store_d16_x:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
index dc5abf418131ce..2c4bd72241d66d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.dwordx3.ll
@@ -1,4 +1,4 @@
-;RUN: llc < %s -march=amdgcn -mcpu=gfx700 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN
+;RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=gfx700 -verify-machineinstrs | FileCheck %s -check-prefixes=GCN
 
 ; GCN-LABEL: {{^}}tbuffer_raw_store_immoffs_x3:
 ; GCN: tbuffer_store_format_xyz v[0:2], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16,BUF_NUM_FORMAT_FLOAT] offset:42
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
index ed31ea3c40feaf..aee01772dabe83 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tbuffer.store.ll
@@ -1,5 +1,5 @@
-;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefixes=GCN,VERDE %s
-;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+;RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck -check-prefixes=GCN,VERDE %s
+;RUN: llc -global-isel=0 < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}tbuffer_store:
 ; GCN: tbuffer_store_format_xyzw v[0:3], off, s[0:3], 0 format:[BUF_DATA_FORMAT_16_16_16_16,BUF_NUM_FORMAT_USCALED]

From 0a3cf7f4762e5608750a8f6b3fa4a6efcba66c27 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 5 Jan 2021 21:33:42 -0500
Subject: [PATCH 12/12] AMDGPU/GlobalISel: Add baseline IR tests for fdiv

The fdiv lowering is currently split between an IR pass and codegen,
so make sure this works end to end. We also currently differ from the
DAG on some edge cases, which this will show in a future change.
---
 .../CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll     | 1171 +++++++++++++++++
 .../CodeGen/AMDGPU/GlobalISel/fdiv.f32.ll     |  651 +++++++++
 .../CodeGen/AMDGPU/GlobalISel/fdiv.f64.ll     |  990 ++++++++++++++
 3 files changed, 2812 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f32.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f64.ll

diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
new file mode 100644
index 00000000000000..7775789bd0d2f5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
@@ -0,0 +1,1171 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; Denormal mode shouldn't matter for f16, check with and without flushing.
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX6,GFX6-IEEE %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX6,GFX6-FLUSH %s
+
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-IEEE %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-FLUSH %s
+
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-IEEE %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-FLUSH %s
+
+define half @v_fdiv_f16(half %a, half %b) {
+; GFX6-IEEE-LABEL: v_fdiv_f16:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_f16:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_fdiv_f16:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GFX89-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX89-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX89-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX89-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX89-NEXT:    v_div_fixup_f16 v0, v2, v1, v0
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv half %a, %b
+  ret half %fdiv
+}
+
+define half @v_fdiv_f16_afn(half %a, half %b) {
+; GFX6-IEEE-LABEL: v_fdiv_f16_afn:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_f16_afn:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_fdiv_f16_afn:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_rcp_f16_e32 v1, v1
+; GFX89-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn half %a, %b
+  ret half %fdiv
+}
+
+define half @v_fdiv_f16_ulp25(half %a, half %b) {
+; GFX6-IEEE-LABEL: v_fdiv_f16_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_f16_ulp25:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_fdiv_f16_ulp25:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GFX89-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX89-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX89-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX89-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX89-NEXT:    v_div_fixup_f16 v0, v2, v1, v0
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv half %a, %b, !fpmath !0
+  ret half %fdiv
+}
+
+define half @v_rcp_f16(half %x) {
+; GFX6-IEEE-LABEL: v_rcp_f16:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, 1.0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v0, v0, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v1, v0, v1
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v0, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_rcp_f16:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, 1.0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v0, v0, v1
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, vcc, v1, v0, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v0, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_rcp_f16:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv half 1.0, %x
+  ret half %fdiv
+}
+
+define half @v_rcp_f16_arcp(half %x) {
+; GFX6-LABEL: v_rcp_f16_arcp:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, 1.0
+; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_rcp_f16_arcp:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp half 1.0, %x
+  ret half %fdiv
+}
+
+define half @v_rcp_f16_arcp_afn(half %x) {
+; GFX6-LABEL: v_rcp_f16_arcp_afn:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, 1.0
+; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
+; GFX6-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_rcp_f16_arcp_afn:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp afn half 1.0, %x
+  ret half %fdiv
+}
+
+define half @v_rcp_f16_ulp25(half %x) {
+; GFX6-IEEE-LABEL: v_rcp_f16_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, 1.0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v0, v0, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v1, v0, v1
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v0, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_rcp_f16_ulp25:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, 1.0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v0, v0, v1
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, vcc, v1, v0, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v0, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_rcp_f16_ulp25:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv half 1.0, %x, !fpmath !0
+  ret half %fdiv
+}
+
+define half @v_fdiv_f16_afn_ulp25(half %a, half %b) {
+; GFX6-IEEE-LABEL: v_fdiv_f16_afn_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_f16_afn_ulp25:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_fdiv_f16_afn_ulp25:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_rcp_f16_e32 v1, v1
+; GFX89-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn half %a, %b, !fpmath !0
+  ret half %fdiv
+}
+
+define half @v_fdiv_f16_arcp_ulp25(half %a, half %b) {
+; GFX6-LABEL: v_fdiv_f16_arcp_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_rcp_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: v_fdiv_f16_arcp_ulp25:
+; GFX89:       ; %bb.0:
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    v_rcp_f16_e32 v1, v1
+; GFX89-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp half %a, %b, !fpmath !0
+  ret half %fdiv
+}
+
+define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
+; GFX6-IEEE-LABEL: v_fdiv_v2f16:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_v2f16:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v5, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX8-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v7, v6
+; GFX8-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX8-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX8-NEXT:    v_mul_f32_e32 v3, v7, v5
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX8-NEXT:    v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_div_fixup_f16 v1, v3, v4, v6
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v5, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX9-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v7, v6
+; GFX9-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX9-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, v7, v5
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX9-NEXT:    v_div_fixup_f16 v0, v2, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_div_fixup_f16 v1, v3, v4, v6
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v2, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x half> %a, %b
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_fdiv_v2f16_afn(<2 x half> %a, <2 x half> %b) {
+; GFX6-IEEE-LABEL: v_fdiv_v2f16_afn:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_v2f16_afn:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f16_afn:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f16_afn:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX9-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v2, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn <2 x half> %a, %b
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
+; GFX6-IEEE-LABEL: v_fdiv_v2f16_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_v2f16_ulp25:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f16_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v5, v4
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX8-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX8-NEXT:    v_cvt_f32_f16_e32 v7, v6
+; GFX8-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX8-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX8-NEXT:    v_mul_f32_e32 v3, v7, v5
+; GFX8-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX8-NEXT:    v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_div_fixup_f16 v1, v3, v4, v6
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f16_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v2, v1
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v5, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v3, v0
+; GFX9-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX9-NEXT:    v_cvt_f32_f16_e32 v7, v6
+; GFX9-NEXT:    v_rcp_f32_e32 v5, v5
+; GFX9-NEXT:    v_mul_f32_e32 v2, v3, v2
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v2, v2
+; GFX9-NEXT:    v_mul_f32_e32 v3, v7, v5
+; GFX9-NEXT:    v_cvt_f16_f32_e32 v3, v3
+; GFX9-NEXT:    v_div_fixup_f16 v0, v2, v1, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_div_fixup_f16 v1, v3, v4, v6
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v2, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x half> %a, %b, !fpmath !0
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
+; GFX6-IEEE-LABEL: v_rcp_v2f16:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v2, 1.0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v3, s[4:5], v0, v0, v2
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v2, v0, v2
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v3, v3, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v3, v0, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, v2
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v2, v1, v2
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v3, v3, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v3, v1, v2
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_rcp_v2f16:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    s_movk_i32 s6, 0x3c00
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v2, s6
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v3, s[4:5], v0, v0, v2
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v2, v0, v2
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v3, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, -v3, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v3, v3, v4, v6
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v4, s6
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v3, v0, v2
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v4
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v4, v1, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v6, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v3, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v6
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v1, v4
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_v2f16:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_v2f16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v1, v0
+; GFX9-NEXT:    v_rcp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x half> <half 1.0, half 1.0>, %x
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_rcp_v2f16_arcp(<2 x half> %x) {
+; GFX6-LABEL: v_rcp_v2f16_arcp:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, 1.0
+; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
+; GFX6-NEXT:    v_rcp_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_f32_e32 v0, v2, v0
+; GFX6-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_v2f16_arcp:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_v2f16_arcp:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v1, v0
+; GFX9-NEXT:    v_rcp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp <2 x half> <half 1.0, half 1.0>, %x
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_rcp_v2f16_arcp_afn(<2 x half> %x) {
+; GFX6-LABEL: v_rcp_v2f16_arcp_afn:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, 1.0
+; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
+; GFX6-NEXT:    v_rcp_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_f32_e32 v0, v2, v0
+; GFX6-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_v2f16_arcp_afn:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_v2f16_arcp_afn:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v1, v0
+; GFX9-NEXT:    v_rcp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp afn <2 x half> <half 1.0, half 1.0>, %x
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
+; GFX6-IEEE-LABEL: v_rcp_v2f16_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v2, 1.0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v3, s[4:5], v0, v0, v2
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v2, v0, v2
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v3, v3, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v3, v0, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, v2
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v2, v1, v2
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, -v3, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v3, v3, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v3, v1, v2
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_rcp_v2f16_ulp25:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    s_movk_i32 s6, 0x3c00
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v2, s6
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v3, s[4:5], v0, v0, v2
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v2, v0, v2
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v3, v4, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v3, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, -v3, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v3, v3, v4, v6
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v4, s6
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v3, v0, v2
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v4
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v4, v1, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v6, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v3, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v6
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v1, v4
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_v2f16_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_v2f16_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v1, v0
+; GFX9-NEXT:    v_rcp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v1, v2, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x half> <half 1.0, half 1.0>, %x, !fpmath !0
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_fdiv_v2f16_afn_ulp25(<2 x half> %a, <2 x half> %b) {
+; GFX6-IEEE-LABEL: v_fdiv_v2f16_afn_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-IEEE-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_v2f16_afn_ulp25:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v3, v3, v1
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v1, v3, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v4, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, v6, v4, v4
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v6, v5
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-FLUSH-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f16_afn_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f16_afn_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX9-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v2, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn <2 x half> %a, %b, !fpmath !0
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_fdiv_v2f16_arcp_ulp25(<2 x half> %a, <2 x half> %b) {
+; GFX6-LABEL: v_fdiv_v2f16_arcp_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX6-NEXT:    v_rcp_f32_e32 v3, v3
+; GFX6-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX6-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f16_arcp_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f16_arcp_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX9-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v2, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp <2 x half> %a, %b, !fpmath !0
+  ret <2 x half> %fdiv
+}
+
+define <2 x half> @v_fdiv_v2f16_arcp_afn_ulp25(<2 x half> %a, <2 x half> %b) {
+; GFX6-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT:    v_rcp_f32_e32 v2, v2
+; GFX6-NEXT:    v_rcp_f32_e32 v3, v3
+; GFX6-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX6-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX8-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT:    v_mov_b32_e32 v1, 16
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_rcp_f16_e32 v2, v1
+; GFX9-NEXT:    v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX9-NEXT:    v_mul_f16_e32 v2, v0, v2
+; GFX9-NEXT:    v_mul_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    v_and_or_b32 v0, v2, v1, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn arcp <2 x half> %a, %b, !fpmath !0
+  ret <2 x half> %fdiv
+}
+
+!0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f32.ll
new file mode 100644
index 00000000000000..02114a058c8910
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f32.ll
@@ -0,0 +1,651 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-IEEE,GFX6,GFX6-IEEE %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-FLUSH,GFX6,GFX6-FLUSH %s
+
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-IEEE,GFX89-IEEE,GFX89,GFX8,GFX8-IEEE %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-FLUSH,GFX89,GFX89-FLUSH,GFX8,GFX8-FLUSH %s
+
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-IEEE,GFX89,GFX89-IEEE,GFX9,GFX9-IEEE %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-FLUSH,GFX89,GFX89-FLUSH,GFX9,GFX9-FLUSH %s
+
+define float @v_fdiv_f32(float %a, float %b) {
+; GFX6-IEEE-LABEL: v_fdiv_f32:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_f32:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-IEEE-LABEL: v_fdiv_f32:
+; GFX89-IEEE:       ; %bb.0:
+; GFX89-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v3, vcc, v0, v1, v0
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX89-IEEE-NEXT:    v_fma_f32 v5, -v2, v4, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v4, v5, v4, v4
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v5, v3, v4
+; GFX89-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v3
+; GFX89-IEEE-NEXT:    v_fma_f32 v5, v6, v4, v5
+; GFX89-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v3
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-FLUSH-LABEL: v_fdiv_f32:
+; GFX89-FLUSH:       ; %bb.0:
+; GFX89-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-FLUSH-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX89-FLUSH-NEXT:    v_div_scale_f32 v3, vcc, v0, v1, v0
+; GFX89-FLUSH-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX89-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX89-FLUSH-NEXT:    v_fma_f32 v5, -v2, v4, 1.0
+; GFX89-FLUSH-NEXT:    v_fma_f32 v4, v5, v4, v4
+; GFX89-FLUSH-NEXT:    v_mul_f32_e32 v5, v3, v4
+; GFX89-FLUSH-NEXT:    v_fma_f32 v6, -v2, v5, v3
+; GFX89-FLUSH-NEXT:    v_fma_f32 v5, v6, v4, v5
+; GFX89-FLUSH-NEXT:    v_fma_f32 v2, -v2, v5, v3
+; GFX89-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
+; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv float %a, %b
+  ret float %fdiv
+}
+
+define float @v_fdiv_f32_afn(float %a, float %b) {
+; GCN-LABEL: v_fdiv_f32_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn float %a, %b
+  ret float %fdiv
+}
+
+define float @v_fdiv_f32_ulp25(float %a, float %b) {
+; GFX6-IEEE-LABEL: v_fdiv_f32_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, v0, v1, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_fdiv_f32_ulp25:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    v_mov_b32_e32 v2, 0x6f800000
+; GCN-FLUSH-NEXT:    v_mov_b32_e32 v3, 0x2f800000
+; GCN-FLUSH-NEXT:    v_cmp_gt_f32_e64 vcc, |v1|, v2
+; GCN-FLUSH-NEXT:    v_cndmask_b32_e32 v2, 1.0, v3, vcc
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v2, v0
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-IEEE-LABEL: v_fdiv_f32_ulp25:
+; GFX89-IEEE:       ; %bb.0:
+; GFX89-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v1, v1, v0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v3, vcc, v0, v1, v0
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v4, v2
+; GFX89-IEEE-NEXT:    v_fma_f32 v5, -v2, v4, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v4, v5, v4, v4
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v5, v3, v4
+; GFX89-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v3
+; GFX89-IEEE-NEXT:    v_fma_f32 v5, v6, v4, v5
+; GFX89-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v3
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv float %a, %b, !fpmath !0
+  ret float %fdiv
+}
+
+define float @v_rcp_f32(float %x) {
+; GFX6-IEEE-LABEL: v_rcp_f32:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v1, s[4:5], v0, v0, 1.0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v2, v1
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v3, vcc, 1.0, v0, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v1, v2, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, v4, v2, v2
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v4, v3, v2
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v1, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v5, v2, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v1, -v1, v4, v3
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v1, v1, v2, v4
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v1, v0, 1.0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_rcp_f32:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-IEEE-LABEL: v_rcp_f32:
+; GFX89-IEEE:       ; %bb.0:
+; GFX89-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v1, s[4:5], v0, v0, 1.0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v2, vcc, 1.0, v0, 1.0
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v3, v1
+; GFX89-IEEE-NEXT:    v_fma_f32 v4, -v1, v3, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v3, v4, v3, v3
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v4, v2, v3
+; GFX89-IEEE-NEXT:    v_fma_f32 v5, -v1, v4, v2
+; GFX89-IEEE-NEXT:    v_fma_f32 v4, v5, v3, v4
+; GFX89-IEEE-NEXT:    v_fma_f32 v1, -v1, v4, v2
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v1, v1, v3, v4
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v1, v0, 1.0
+; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv float 1.0, %x
+  ret float %fdiv
+}
+
+define float @v_rcp_f32_arcp(float %x) {
+; GCN-LABEL: v_rcp_f32_arcp:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp float 1.0, %x
+  ret float %fdiv
+}
+
+define float @v_rcp_f32_arcp_afn(float %x) {
+; GCN-LABEL: v_rcp_f32_arcp_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp afn float 1.0, %x
+  ret float %fdiv
+}
+
+define float @v_rcp_f32_ulp25(float %x) {
+; GCN-IEEE-LABEL: v_rcp_f32_ulp25:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_mov_b32_e32 v1, 0x6f800000
+; GCN-IEEE-NEXT:    v_mov_b32_e32 v2, 0x2f800000
+; GCN-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, v1
+; GCN-IEEE-NEXT:    v_cndmask_b32_e32 v1, 1.0, v2, vcc
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, v1, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_rcp_f32_ulp25:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv float 1.0, %x, !fpmath !0
+  ret float %fdiv
+}
+
+define float @v_fdiv_f32_afn_ulp25(float %a, float %b) {
+; GCN-LABEL: v_fdiv_f32_afn_ulp25:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn float %a, %b, !fpmath !0
+  ret float %fdiv
+}
+
+define float @v_fdiv_f32_arcp_ulp25(float %a, float %b) {
+; GCN-IEEE-LABEL: v_fdiv_f32_arcp_ulp25:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_fdiv_f32_arcp_ulp25:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    v_mov_b32_e32 v2, 0x6f800000
+; GCN-FLUSH-NEXT:    v_mov_b32_e32 v3, 0x2f800000
+; GCN-FLUSH-NEXT:    v_cmp_gt_f32_e64 vcc, |v1|, v2
+; GCN-FLUSH-NEXT:    v_cndmask_b32_e32 v2, 1.0, v3, vcc
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v2, v0
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp float %a, %b, !fpmath !0
+  ret float %fdiv
+}
+
+define <2 x float> @v_fdiv_v2f32(<2 x float> %a, <2 x float> %b) {
+; GFX6-IEEE-LABEL: v_fdiv_v2f32:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, s[4:5], v3, v3, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, vcc, v1, v3, v1
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v5, v6, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v4, v6, v6
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v2, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v5, v6, v2
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v5, v6, v2
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-FLUSH-LABEL: v_fdiv_v2f32:
+; GFX6-FLUSH:       ; %bb.0:
+; GFX6-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-FLUSH-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v5, s[4:5], v3, v3, v1
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-FLUSH-NEXT:    v_rcp_f32_e32 v6, v5
+; GFX6-FLUSH-NEXT:    v_div_scale_f32 v2, vcc, v1, v3, v1
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, -v5, v6, 1.0
+; GFX6-FLUSH-NEXT:    v_fma_f32 v4, v4, v6, v6
+; GFX6-FLUSH-NEXT:    v_mul_f32_e32 v6, v2, v4
+; GFX6-FLUSH-NEXT:    v_fma_f32 v7, -v5, v6, v2
+; GFX6-FLUSH-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-FLUSH-NEXT:    v_fma_f32 v2, -v5, v6, v2
+; GFX6-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX6-FLUSH-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-IEEE-LABEL: v_fdiv_v2f32:
+; GFX89-IEEE:       ; %bb.0:
+; GFX89-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v5, s[4:5], v3, v3, v1
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v7, s[4:5], v1, v3, v1
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v8, v4
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v9, v5
+; GFX89-IEEE-NEXT:    v_fma_f32 v10, -v4, v8, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v11, -v5, v9, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v8, v10, v8, v8
+; GFX89-IEEE-NEXT:    v_fma_f32 v9, v11, v9, v9
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v10, v6, v8
+; GFX89-IEEE-NEXT:    v_fma_f32 v12, -v4, v10, v6
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v11, v7, v9
+; GFX89-IEEE-NEXT:    v_fma_f32 v13, -v5, v11, v7
+; GFX89-IEEE-NEXT:    v_fma_f32 v10, v12, v8, v10
+; GFX89-IEEE-NEXT:    v_fma_f32 v4, -v4, v10, v6
+; GFX89-IEEE-NEXT:    v_fma_f32 v11, v13, v9, v11
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v8, v10
+; GFX89-IEEE-NEXT:    v_fma_f32 v5, -v5, v11, v7
+; GFX89-IEEE-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v5, v5, v9, v11
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v1, v5, v3, v1
+; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-FLUSH-LABEL: v_fdiv_v2f32:
+; GFX89-FLUSH:       ; %bb.0:
+; GFX89-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-FLUSH-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX89-FLUSH-NEXT:    v_div_scale_f32 v5, vcc, v0, v2, v0
+; GFX89-FLUSH-NEXT:    v_rcp_f32_e32 v6, v4
+; GFX89-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX89-FLUSH-NEXT:    v_fma_f32 v7, -v4, v6, 1.0
+; GFX89-FLUSH-NEXT:    v_fma_f32 v6, v7, v6, v6
+; GFX89-FLUSH-NEXT:    v_mul_f32_e32 v7, v5, v6
+; GFX89-FLUSH-NEXT:    v_fma_f32 v8, -v4, v7, v5
+; GFX89-FLUSH-NEXT:    v_fma_f32 v7, v8, v6, v7
+; GFX89-FLUSH-NEXT:    v_fma_f32 v4, -v4, v7, v5
+; GFX89-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v4, v4, v6, v7
+; GFX89-FLUSH-NEXT:    v_div_scale_f32 v5, s[4:5], v3, v3, v1
+; GFX89-FLUSH-NEXT:    v_div_scale_f32 v6, vcc, v1, v3, v1
+; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX89-FLUSH-NEXT:    v_rcp_f32_e32 v7, v5
+; GFX89-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX89-FLUSH-NEXT:    v_fma_f32 v2, -v5, v7, 1.0
+; GFX89-FLUSH-NEXT:    v_fma_f32 v2, v2, v7, v7
+; GFX89-FLUSH-NEXT:    v_mul_f32_e32 v4, v6, v2
+; GFX89-FLUSH-NEXT:    v_fma_f32 v7, -v5, v4, v6
+; GFX89-FLUSH-NEXT:    v_fma_f32 v4, v7, v2, v4
+; GFX89-FLUSH-NEXT:    v_fma_f32 v5, -v5, v4, v6
+; GFX89-FLUSH-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GFX89-FLUSH-NEXT:    v_div_fmas_f32 v2, v5, v2, v4
+; GFX89-FLUSH-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX89-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x float> %a, %b
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_fdiv_v2f32_afn(<2 x float> %a, <2 x float> %b) {
+; GCN-LABEL: v_fdiv_v2f32_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v2, v2
+; GCN-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn <2 x float> %a, %b
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_fdiv_v2f32_ulp25(<2 x float> %a, <2 x float> %b) {
+; GFX6-IEEE-LABEL: v_fdiv_v2f32_ulp25:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v5, v4
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v8, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v4, v7, v6
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v5, v7
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v5, s[4:5], v3, v3, v1
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, vcc, v1, v3, v1
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, -v5, v6, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v4, v6, v6
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v6, v2, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v7, -v5, v6, v2
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v5, v6, v2
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v6
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v2, v3, v1
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_fdiv_v2f32_ulp25:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    s_mov_b32 s4, 0x6f800000
+; GCN-FLUSH-NEXT:    v_mov_b32_e32 v4, 0x2f800000
+; GCN-FLUSH-NEXT:    v_cmp_gt_f32_e64 vcc, |v2|, s4
+; GCN-FLUSH-NEXT:    v_cndmask_b32_e32 v5, 1.0, v4, vcc
+; GCN-FLUSH-NEXT:    v_cmp_gt_f32_e64 vcc, |v3|, s4
+; GCN-FLUSH-NEXT:    v_cndmask_b32_e32 v4, 1.0, v4, vcc
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v2, v2, v5
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v3, v3, v4
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v2, v2
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v5, v0
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v1, v4, v1
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-IEEE-LABEL: v_fdiv_v2f32_ulp25:
+; GFX89-IEEE:       ; %bb.0:
+; GFX89-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v4, s[4:5], v2, v2, v0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v5, s[4:5], v3, v3, v1
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v6, vcc, v0, v2, v0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v7, s[4:5], v1, v3, v1
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v8, v4
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v9, v5
+; GFX89-IEEE-NEXT:    v_fma_f32 v10, -v4, v8, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v11, -v5, v9, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v8, v10, v8, v8
+; GFX89-IEEE-NEXT:    v_fma_f32 v9, v11, v9, v9
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v10, v6, v8
+; GFX89-IEEE-NEXT:    v_fma_f32 v12, -v4, v10, v6
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v11, v7, v9
+; GFX89-IEEE-NEXT:    v_fma_f32 v13, -v5, v11, v7
+; GFX89-IEEE-NEXT:    v_fma_f32 v10, v12, v8, v10
+; GFX89-IEEE-NEXT:    v_fma_f32 v4, -v4, v10, v6
+; GFX89-IEEE-NEXT:    v_fma_f32 v11, v13, v9, v11
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v4, v4, v8, v10
+; GFX89-IEEE-NEXT:    v_fma_f32 v5, -v5, v11, v7
+; GFX89-IEEE-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v5, v5, v9, v11
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v4, v2, v0
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v1, v5, v3, v1
+; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x float> %a, %b, !fpmath !0
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_rcp_v2f32(<2 x float> %x) {
+; GFX6-IEEE-LABEL: v_rcp_v2f32:
+; GFX6-IEEE:       ; %bb.0:
+; GFX6-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
+; GFX6-IEEE-NEXT:    v_rcp_f32_e32 v4, v3
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v0, 1.0
+; GFX6-IEEE-NEXT:    v_div_scale_f32 v2, vcc, 1.0, v1, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, -v3, v4, 1.0
+; GFX6-IEEE-NEXT:    v_fma_f32 v4, v5, v4, v4
+; GFX6-IEEE-NEXT:    v_mul_f32_e32 v5, v2, v4
+; GFX6-IEEE-NEXT:    v_fma_f32 v6, -v3, v5, v2
+; GFX6-IEEE-NEXT:    v_fma_f32 v5, v6, v4, v5
+; GFX6-IEEE-NEXT:    v_fma_f32 v2, -v3, v5, v2
+; GFX6-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v4, v5
+; GFX6-IEEE-NEXT:    v_div_fixup_f32 v1, v2, v1, 1.0
+; GFX6-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_rcp_v2f32:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-IEEE-LABEL: v_rcp_v2f32:
+; GFX89-IEEE:       ; %bb.0:
+; GFX89-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
+; GFX89-IEEE-NEXT:    v_div_scale_f32 v5, s[4:5], 1.0, v1, 1.0
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v6, v2
+; GFX89-IEEE-NEXT:    v_rcp_f32_e32 v7, v3
+; GFX89-IEEE-NEXT:    v_fma_f32 v8, -v2, v6, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v9, -v3, v7, 1.0
+; GFX89-IEEE-NEXT:    v_fma_f32 v6, v8, v6, v6
+; GFX89-IEEE-NEXT:    v_fma_f32 v7, v9, v7, v7
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v8, v4, v6
+; GFX89-IEEE-NEXT:    v_fma_f32 v10, -v2, v8, v4
+; GFX89-IEEE-NEXT:    v_mul_f32_e32 v9, v5, v7
+; GFX89-IEEE-NEXT:    v_fma_f32 v11, -v3, v9, v5
+; GFX89-IEEE-NEXT:    v_fma_f32 v8, v10, v6, v8
+; GFX89-IEEE-NEXT:    v_fma_f32 v2, -v2, v8, v4
+; GFX89-IEEE-NEXT:    v_fma_f32 v9, v11, v7, v9
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v2, v2, v6, v8
+; GFX89-IEEE-NEXT:    v_fma_f32 v3, -v3, v9, v5
+; GFX89-IEEE-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX89-IEEE-NEXT:    v_div_fmas_f32 v3, v3, v7, v9
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v0, v2, v0, 1.0
+; GFX89-IEEE-NEXT:    v_div_fixup_f32 v1, v3, v1, 1.0
+; GFX89-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x float> <float 1.0, float 1.0>, %x
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_rcp_v2f32_arcp(<2 x float> %x) {
+; GCN-LABEL: v_rcp_v2f32_arcp:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp <2 x float> <float 1.0, float 1.0>, %x
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_rcp_v2f32_arcp_afn(<2 x float> %x) {
+; GCN-LABEL: v_rcp_v2f32_arcp_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp afn <2 x float> <float 1.0, float 1.0>, %x
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_rcp_v2f32_ulp25(<2 x float> %x) {
+; GCN-IEEE-LABEL: v_rcp_v2f32_ulp25:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b32 s4, 0x6f800000
+; GCN-IEEE-NEXT:    v_mov_b32_e32 v2, 0x2f800000
+; GCN-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, |v0|, s4
+; GCN-IEEE-NEXT:    v_cndmask_b32_e32 v3, 1.0, v2, vcc
+; GCN-IEEE-NEXT:    v_cmp_gt_f32_e64 vcc, |v1|, s4
+; GCN-IEEE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v3
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, 1.0, v0
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v1, 1.0, v1
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, v3, v0
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_rcp_v2f32_ulp25:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v1, v1
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x float> <float 1.0, float 1.0>, %x, !fpmath !0
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_fdiv_v2f32_afn_ulp25(<2 x float> %a, <2 x float> %b) {
+; GCN-LABEL: v_fdiv_v2f32_afn_ulp25:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v2, v2
+; GCN-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn <2 x float> %a, %b, !fpmath !0
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_fdiv_v2f32_arcp_ulp25(<2 x float> %a, <2 x float> %b) {
+; GCN-IEEE-LABEL: v_fdiv_v2f32_arcp_ulp25:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v2, v2
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-FLUSH-LABEL: v_fdiv_v2f32_arcp_ulp25:
+; GCN-FLUSH:       ; %bb.0:
+; GCN-FLUSH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-FLUSH-NEXT:    s_mov_b32 s4, 0x6f800000
+; GCN-FLUSH-NEXT:    v_mov_b32_e32 v4, 0x2f800000
+; GCN-FLUSH-NEXT:    v_cmp_gt_f32_e64 vcc, |v2|, s4
+; GCN-FLUSH-NEXT:    v_cndmask_b32_e32 v5, 1.0, v4, vcc
+; GCN-FLUSH-NEXT:    v_cmp_gt_f32_e64 vcc, |v3|, s4
+; GCN-FLUSH-NEXT:    v_cndmask_b32_e32 v4, 1.0, v4, vcc
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v2, v2, v5
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v3, v3, v4
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v2, v2
+; GCN-FLUSH-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v0, v5, v0
+; GCN-FLUSH-NEXT:    v_mul_f32_e32 v1, v4, v1
+; GCN-FLUSH-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp <2 x float> %a, %b, !fpmath !0
+  ret <2 x float> %fdiv
+}
+
+define <2 x float> @v_fdiv_v2f32_arcp_afn_ulp25(<2 x float> %a, <2 x float> %b) {
+; GCN-LABEL: v_fdiv_v2f32_arcp_afn_ulp25:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f32_e32 v2, v2
+; GCN-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn arcp <2 x float> %a, %b, !fpmath !0
+  ret <2 x float> %fdiv
+}
+
+!0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f64.ll
new file mode 100644
index 00000000000000..4216892e1dd56a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f64.ll
@@ -0,0 +1,990 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX6 %s
+
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+define double @v_fdiv_f64(double %a, double %b) {
+; GFX6-LABEL: v_fdiv_f64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX6-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[0:1], v[2:3], v[0:1]
+; GFX6-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v11
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v3, v5
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_mul_f64 v[8:9], v[10:11], v[6:7]
+; GFX6-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[8:9], v[10:11]
+; GFX6-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX8-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX8-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX8-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[6:7], 1.0
+; GFX8-NEXT:    v_fma_f64 v[6:7], v[6:7], v[10:11], v[6:7]
+; GFX8-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
+; GFX8-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
+; GFX8-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX9-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX9-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX9-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[6:7], 1.0
+; GFX9-NEXT:    v_fma_f64 v[6:7], v[6:7], v[10:11], v[6:7]
+; GFX9-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
+; GFX9-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
+; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv double %a, %b
+  ret double %fdiv
+}
+
+define double @v_fdiv_f64_afn(double %a, double %b) {
+; GCN-LABEL: v_fdiv_f64_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f64_e32 v[2:3], v[2:3]
+; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn double %a, %b
+  ret double %fdiv
+}
+
+define double @v_fdiv_f64_ulp25(double %a, double %b) {
+; GFX6-LABEL: v_fdiv_f64_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX6-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[0:1], v[2:3], v[0:1]
+; GFX6-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v11
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v3, v5
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_mul_f64 v[8:9], v[10:11], v[6:7]
+; GFX6-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[8:9], v[10:11]
+; GFX6-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_f64_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX8-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX8-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX8-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[6:7], 1.0
+; GFX8-NEXT:    v_fma_f64 v[6:7], v[6:7], v[10:11], v[6:7]
+; GFX8-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
+; GFX8-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
+; GFX8-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_f64_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX9-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX9-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX9-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[6:7], 1.0
+; GFX9-NEXT:    v_fma_f64 v[6:7], v[6:7], v[10:11], v[6:7]
+; GFX9-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
+; GFX9-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
+; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv double %a, %b, !fpmath !0
+  ret double %fdiv
+}
+
+define double @v_rcp_f64(double %x) {
+; GFX6-LABEL: v_rcp_f64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], 1.0, v[0:1], 1.0
+; GFX6-NEXT:    v_mov_b32_e32 v10, 0x3ff00000
+; GFX6-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v10, v9
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v1, v3
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-NEXT:    v_mul_f64 v[6:7], v[8:9], v[4:5]
+; GFX6-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[6:7], v[8:9]
+; GFX6-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX8-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX8-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX8-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-NEXT:    v_div_scale_f64 v[6:7], vcc, 1.0, v[0:1], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[4:5], 1.0
+; GFX8-NEXT:    v_fma_f64 v[4:5], v[4:5], v[8:9], v[4:5]
+; GFX8-NEXT:    v_mul_f64 v[8:9], v[6:7], v[4:5]
+; GFX8-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[8:9], v[6:7]
+; GFX8-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX9-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX9-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX9-NEXT:    v_div_scale_f64 v[6:7], vcc, 1.0, v[0:1], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[4:5], 1.0
+; GFX9-NEXT:    v_fma_f64 v[4:5], v[4:5], v[8:9], v[4:5]
+; GFX9-NEXT:    v_mul_f64 v[8:9], v[6:7], v[4:5]
+; GFX9-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[8:9], v[6:7]
+; GFX9-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv double 1.0, %x
+  ret double %fdiv
+}
+
+define double @v_rcp_f64_arcp(double %x) {
+; GFX6-LABEL: v_rcp_f64_arcp:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], 1.0, v[0:1], 1.0
+; GFX6-NEXT:    v_mov_b32_e32 v10, 0x3ff00000
+; GFX6-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v10, v9
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v1, v3
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-NEXT:    v_mul_f64 v[6:7], v[8:9], v[4:5]
+; GFX6-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[6:7], v[8:9]
+; GFX6-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_f64_arcp:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX8-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX8-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX8-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-NEXT:    v_div_scale_f64 v[6:7], vcc, 1.0, v[0:1], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[4:5], 1.0
+; GFX8-NEXT:    v_fma_f64 v[4:5], v[4:5], v[8:9], v[4:5]
+; GFX8-NEXT:    v_mul_f64 v[8:9], v[6:7], v[4:5]
+; GFX8-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[8:9], v[6:7]
+; GFX8-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_f64_arcp:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX9-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX9-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX9-NEXT:    v_div_scale_f64 v[6:7], vcc, 1.0, v[0:1], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[4:5], 1.0
+; GFX9-NEXT:    v_fma_f64 v[4:5], v[4:5], v[8:9], v[4:5]
+; GFX9-NEXT:    v_mul_f64 v[8:9], v[6:7], v[4:5]
+; GFX9-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[8:9], v[6:7]
+; GFX9-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp double 1.0, %x
+  ret double %fdiv
+}
+
+define double @v_rcp_f64_arcp_afn(double %x) {
+; GCN-LABEL: v_rcp_f64_arcp_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f64_e32 v[0:1], v[0:1]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp afn double 1.0, %x
+  ret double %fdiv
+}
+
+define double @v_rcp_f64_ulp25(double %x) {
+; GFX6-LABEL: v_rcp_f64_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], 1.0, v[0:1], 1.0
+; GFX6-NEXT:    v_mov_b32_e32 v10, 0x3ff00000
+; GFX6-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v10, v9
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v1, v3
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-NEXT:    v_mul_f64 v[6:7], v[8:9], v[4:5]
+; GFX6-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[6:7], v[8:9]
+; GFX6-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_f64_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX8-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX8-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX8-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-NEXT:    v_div_scale_f64 v[6:7], vcc, 1.0, v[0:1], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[4:5], 1.0
+; GFX8-NEXT:    v_fma_f64 v[4:5], v[4:5], v[8:9], v[4:5]
+; GFX8-NEXT:    v_mul_f64 v[8:9], v[6:7], v[4:5]
+; GFX8-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[8:9], v[6:7]
+; GFX8-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_f64_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[2:3], s[4:5], v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_rcp_f64_e32 v[4:5], v[2:3]
+; GFX9-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0
+; GFX9-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX9-NEXT:    v_div_scale_f64 v[6:7], vcc, 1.0, v[0:1], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[4:5], 1.0
+; GFX9-NEXT:    v_fma_f64 v[4:5], v[4:5], v[8:9], v[4:5]
+; GFX9-NEXT:    v_mul_f64 v[8:9], v[6:7], v[4:5]
+; GFX9-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[8:9], v[6:7]
+; GFX9-NEXT:    v_div_fmas_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[2:3], v[0:1], 1.0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv double 1.0, %x, !fpmath !0
+  ret double %fdiv
+}
+
+define double @v_fdiv_f64_afn_ulp25(double %a, double %b) {
+; GCN-LABEL: v_fdiv_f64_afn_ulp25:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f64_e32 v[2:3], v[2:3]
+; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn double %a, %b, !fpmath !0
+  ret double %fdiv
+}
+
+define double @v_fdiv_f64_arcp_ulp25(double %a, double %b) {
+; GFX6-LABEL: v_fdiv_f64_arcp_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX6-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[0:1], v[2:3], v[0:1]
+; GFX6-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v11
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v3, v5
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_mul_f64 v[8:9], v[10:11], v[6:7]
+; GFX6-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[8:9], v[10:11]
+; GFX6-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_f64_arcp_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX8-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX8-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX8-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[6:7], 1.0
+; GFX8-NEXT:    v_fma_f64 v[6:7], v[6:7], v[10:11], v[6:7]
+; GFX8-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
+; GFX8-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
+; GFX8-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_f64_arcp_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[2:3], v[2:3], v[0:1]
+; GFX9-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX9-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX9-NEXT:    v_div_scale_f64 v[8:9], vcc, v[0:1], v[2:3], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[6:7], 1.0
+; GFX9-NEXT:    v_fma_f64 v[6:7], v[6:7], v[10:11], v[6:7]
+; GFX9-NEXT:    v_mul_f64 v[10:11], v[8:9], v[6:7]
+; GFX9-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[10:11], v[8:9]
+; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp double %a, %b, !fpmath !0
+  ret double %fdiv
+}
+
+define <2 x double> @v_fdiv_v2f64(<2 x double> %a, <2 x double> %b) {
+; GFX6-LABEL: v_fdiv_v2f64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX6-NEXT:    v_div_scale_f64 v[14:15], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX6-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[0:1], v[4:5], v[0:1]
+; GFX6-NEXT:    v_rcp_f64_e32 v[10:11], v[8:9]
+; GFX6-NEXT:    v_rcp_f64_e32 v[16:17], v[14:15]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v19
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v9
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v7, v15
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX6-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[14:15], v[16:17], 1.0
+; GFX6-NEXT:    v_fma_f64 v[12:13], v[16:17], v[12:13], v[16:17]
+; GFX6-NEXT:    v_mul_f64 v[16:17], v[18:19], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[16:17], v[18:19]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[14:15], v[12:13], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[10:11], v[18:19], v[10:11], v[16:17]
+; GFX6-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[12:13]
+; GFX6-NEXT:    v_div_scale_f64 v[12:13], s[6:7], v[2:3], v[6:7], v[2:3]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[10:11], v[4:5], v[0:1]
+; GFX6-NEXT:    v_mul_f64 v[16:17], v[12:13], v[8:9]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v13
+; GFX6-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[16:17], v[12:13]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    s_nop 1
+; GFX6-NEXT:    v_div_fmas_f64 v[8:9], v[18:19], v[8:9], v[16:17]
+; GFX6-NEXT:    v_div_fixup_f64 v[2:3], v[8:9], v[6:7], v[2:3]
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX8-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX8-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX8-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX8-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
+; GFX8-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX8-NEXT:    v_div_scale_f64 v[18:19], vcc, v[0:1], v[4:5], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX8-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[10:11], v[14:15], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], v[14:15], v[16:17], v[14:15]
+; GFX8-NEXT:    v_mul_f64 v[16:17], v[18:19], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[16:17], v[18:19]
+; GFX8-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[2:3], v[6:7], v[2:3]
+; GFX8-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[12:13], v[16:17]
+; GFX8-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX8-NEXT:    v_mul_f64 v[20:21], v[18:19], v[14:15]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[8:9], v[4:5], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[10:11], -v[10:11], v[20:21], v[18:19]
+; GFX8-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
+; GFX8-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX9-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX9-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX9-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX9-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
+; GFX9-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX9-NEXT:    v_div_scale_f64 v[18:19], vcc, v[0:1], v[4:5], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX9-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[10:11], v[14:15], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], v[14:15], v[16:17], v[14:15]
+; GFX9-NEXT:    v_mul_f64 v[16:17], v[18:19], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[16:17], v[18:19]
+; GFX9-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[2:3], v[6:7], v[2:3]
+; GFX9-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[12:13], v[16:17]
+; GFX9-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX9-NEXT:    v_mul_f64 v[20:21], v[18:19], v[14:15]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[8:9], v[4:5], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[10:11], -v[10:11], v[20:21], v[18:19]
+; GFX9-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
+; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x double> %a, %b
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_fdiv_v2f64_afn(<2 x double> %a, <2 x double> %b) {
+; GCN-LABEL: v_fdiv_v2f64_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[4:5]
+; GCN-NEXT:    v_rcp_f64_e32 v[6:7], v[6:7]
+; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GCN-NEXT:    v_mul_f64 v[2:3], v[2:3], v[6:7]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn <2 x double> %a, %b
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_fdiv_v2f64_ulp25(<2 x double> %a, <2 x double> %b) {
+; GFX6-LABEL: v_fdiv_v2f64_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX6-NEXT:    v_div_scale_f64 v[14:15], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX6-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[0:1], v[4:5], v[0:1]
+; GFX6-NEXT:    v_rcp_f64_e32 v[10:11], v[8:9]
+; GFX6-NEXT:    v_rcp_f64_e32 v[16:17], v[14:15]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v19
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v9
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v7, v15
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX6-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[14:15], v[16:17], 1.0
+; GFX6-NEXT:    v_fma_f64 v[12:13], v[16:17], v[12:13], v[16:17]
+; GFX6-NEXT:    v_mul_f64 v[16:17], v[18:19], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[16:17], v[18:19]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[14:15], v[12:13], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[10:11], v[18:19], v[10:11], v[16:17]
+; GFX6-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[12:13]
+; GFX6-NEXT:    v_div_scale_f64 v[12:13], s[6:7], v[2:3], v[6:7], v[2:3]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[10:11], v[4:5], v[0:1]
+; GFX6-NEXT:    v_mul_f64 v[16:17], v[12:13], v[8:9]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v13
+; GFX6-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[16:17], v[12:13]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    s_nop 1
+; GFX6-NEXT:    v_div_fmas_f64 v[8:9], v[18:19], v[8:9], v[16:17]
+; GFX6-NEXT:    v_div_fixup_f64 v[2:3], v[8:9], v[6:7], v[2:3]
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f64_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX8-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX8-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX8-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX8-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
+; GFX8-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX8-NEXT:    v_div_scale_f64 v[18:19], vcc, v[0:1], v[4:5], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX8-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[10:11], v[14:15], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], v[14:15], v[16:17], v[14:15]
+; GFX8-NEXT:    v_mul_f64 v[16:17], v[18:19], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[16:17], v[18:19]
+; GFX8-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[2:3], v[6:7], v[2:3]
+; GFX8-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[12:13], v[16:17]
+; GFX8-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX8-NEXT:    v_mul_f64 v[20:21], v[18:19], v[14:15]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[8:9], v[4:5], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[10:11], -v[10:11], v[20:21], v[18:19]
+; GFX8-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
+; GFX8-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f64_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX9-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX9-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX9-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX9-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
+; GFX9-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX9-NEXT:    v_div_scale_f64 v[18:19], vcc, v[0:1], v[4:5], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX9-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[10:11], v[14:15], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], v[14:15], v[16:17], v[14:15]
+; GFX9-NEXT:    v_mul_f64 v[16:17], v[18:19], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[16:17], v[18:19]
+; GFX9-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[2:3], v[6:7], v[2:3]
+; GFX9-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[12:13], v[16:17]
+; GFX9-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX9-NEXT:    v_mul_f64 v[20:21], v[18:19], v[14:15]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[8:9], v[4:5], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[10:11], -v[10:11], v[20:21], v[18:19]
+; GFX9-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
+; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x double> %a, %b, !fpmath !0
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_rcp_v2f64(<2 x double> %x) {
+; GFX6-LABEL: v_rcp_v2f64:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX6-NEXT:    v_div_scale_f64 v[10:11], s[4:5], 1.0, v[0:1], 1.0
+; GFX6-NEXT:    v_mov_b32_e32 v18, 0x3ff00000
+; GFX6-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v18, v11
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[2:3], v[2:3], 1.0
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v1, v5
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[12:13], v[6:7]
+; GFX6-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX6-NEXT:    v_mul_f64 v[14:15], v[10:11], v[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[14:15], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[6:7], v[10:11], v[6:7], v[14:15]
+; GFX6-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX6-NEXT:    v_div_scale_f64 v[16:17], s[6:7], 1.0, v[2:3], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], -v[8:9], v[12:13], 1.0
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v3, v9
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[12:13]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v18, v17
+; GFX6-NEXT:    v_mul_f64 v[12:13], v[16:17], v[4:5]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], -v[8:9], v[12:13], v[16:17]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[6:7], v[0:1], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[4:5], v[10:11], v[4:5], v[12:13]
+; GFX6-NEXT:    v_div_fixup_f64 v[2:3], v[4:5], v[2:3], 1.0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_v2f64:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX8-NEXT:    v_div_scale_f64 v[6:7], s[4:5], v[2:3], v[2:3], 1.0
+; GFX8-NEXT:    v_div_scale_f64 v[16:17], s[4:5], 1.0, v[2:3], 1.0
+; GFX8-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GFX8-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GFX8-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-NEXT:    v_div_scale_f64 v[12:13], vcc, 1.0, v[0:1], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], -v[4:5], v[8:9], 1.0
+; GFX8-NEXT:    v_fma_f64 v[18:19], -v[6:7], v[10:11], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[8:9]
+; GFX8-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX8-NEXT:    v_mul_f64 v[14:15], v[12:13], v[8:9]
+; GFX8-NEXT:    v_mul_f64 v[18:19], v[16:17], v[10:11]
+; GFX8-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[14:15], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[18:19], v[16:17]
+; GFX8-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[8:9], v[14:15]
+; GFX8-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX8-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[18:19]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
+; GFX8-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_v2f64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_div_scale_f64 v[6:7], s[4:5], v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_div_scale_f64 v[16:17], s[4:5], 1.0, v[2:3], 1.0
+; GFX9-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GFX9-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GFX9-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX9-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX9-NEXT:    v_div_scale_f64 v[12:13], vcc, 1.0, v[0:1], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], -v[4:5], v[8:9], 1.0
+; GFX9-NEXT:    v_fma_f64 v[18:19], -v[6:7], v[10:11], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[8:9]
+; GFX9-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX9-NEXT:    v_mul_f64 v[14:15], v[12:13], v[8:9]
+; GFX9-NEXT:    v_mul_f64 v[18:19], v[16:17], v[10:11]
+; GFX9-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[14:15], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[18:19], v[16:17]
+; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[8:9], v[14:15]
+; GFX9-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX9-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[18:19]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
+; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x double> <double 1.0, double 1.0>, %x
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_rcp_v2f64_arcp(<2 x double> %x) {
+; GFX6-LABEL: v_rcp_v2f64_arcp:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX6-NEXT:    v_div_scale_f64 v[10:11], s[4:5], 1.0, v[0:1], 1.0
+; GFX6-NEXT:    v_mov_b32_e32 v18, 0x3ff00000
+; GFX6-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v18, v11
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[2:3], v[2:3], 1.0
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v1, v5
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[12:13], v[6:7]
+; GFX6-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX6-NEXT:    v_mul_f64 v[14:15], v[10:11], v[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[14:15], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[6:7], v[10:11], v[6:7], v[14:15]
+; GFX6-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX6-NEXT:    v_div_scale_f64 v[16:17], s[6:7], 1.0, v[2:3], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], -v[8:9], v[12:13], 1.0
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v3, v9
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[12:13]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v18, v17
+; GFX6-NEXT:    v_mul_f64 v[12:13], v[16:17], v[4:5]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], -v[8:9], v[12:13], v[16:17]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[6:7], v[0:1], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[4:5], v[10:11], v[4:5], v[12:13]
+; GFX6-NEXT:    v_div_fixup_f64 v[2:3], v[4:5], v[2:3], 1.0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_v2f64_arcp:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX8-NEXT:    v_div_scale_f64 v[6:7], s[4:5], v[2:3], v[2:3], 1.0
+; GFX8-NEXT:    v_div_scale_f64 v[16:17], s[4:5], 1.0, v[2:3], 1.0
+; GFX8-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GFX8-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GFX8-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-NEXT:    v_div_scale_f64 v[12:13], vcc, 1.0, v[0:1], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], -v[4:5], v[8:9], 1.0
+; GFX8-NEXT:    v_fma_f64 v[18:19], -v[6:7], v[10:11], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[8:9]
+; GFX8-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX8-NEXT:    v_mul_f64 v[14:15], v[12:13], v[8:9]
+; GFX8-NEXT:    v_mul_f64 v[18:19], v[16:17], v[10:11]
+; GFX8-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[14:15], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[18:19], v[16:17]
+; GFX8-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[8:9], v[14:15]
+; GFX8-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX8-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[18:19]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
+; GFX8-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_v2f64_arcp:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_div_scale_f64 v[6:7], s[4:5], v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_div_scale_f64 v[16:17], s[4:5], 1.0, v[2:3], 1.0
+; GFX9-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GFX9-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GFX9-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX9-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX9-NEXT:    v_div_scale_f64 v[12:13], vcc, 1.0, v[0:1], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], -v[4:5], v[8:9], 1.0
+; GFX9-NEXT:    v_fma_f64 v[18:19], -v[6:7], v[10:11], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[8:9]
+; GFX9-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX9-NEXT:    v_mul_f64 v[14:15], v[12:13], v[8:9]
+; GFX9-NEXT:    v_mul_f64 v[18:19], v[16:17], v[10:11]
+; GFX9-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[14:15], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[18:19], v[16:17]
+; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[8:9], v[14:15]
+; GFX9-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX9-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[18:19]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
+; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp <2 x double> <double 1.0, double 1.0>, %x
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_rcp_v2f64_arcp_afn(<2 x double> %x) {
+; GCN-LABEL: v_rcp_v2f64_arcp_afn:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f64_e32 v[0:1], v[0:1]
+; GCN-NEXT:    v_rcp_f64_e32 v[2:3], v[2:3]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp afn <2 x double> <double 1.0, double 1.0>, %x
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_rcp_v2f64_ulp25(<2 x double> %x) {
+; GFX6-LABEL: v_rcp_v2f64_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX6-NEXT:    v_div_scale_f64 v[10:11], s[4:5], 1.0, v[0:1], 1.0
+; GFX6-NEXT:    v_mov_b32_e32 v18, 0x3ff00000
+; GFX6-NEXT:    v_rcp_f64_e32 v[6:7], v[4:5]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v18, v11
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7]
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[2:3], v[2:3], 1.0
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[6:7], 1.0
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v1, v5
+; GFX6-NEXT:    v_fma_f64 v[6:7], v[6:7], v[12:13], v[6:7]
+; GFX6-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX6-NEXT:    v_mul_f64 v[14:15], v[10:11], v[6:7]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], -v[4:5], v[14:15], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[6:7], v[10:11], v[6:7], v[14:15]
+; GFX6-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX6-NEXT:    v_div_scale_f64 v[16:17], s[6:7], 1.0, v[2:3], 1.0
+; GFX6-NEXT:    v_fma_f64 v[4:5], -v[8:9], v[12:13], 1.0
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v3, v9
+; GFX6-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[12:13]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v18, v17
+; GFX6-NEXT:    v_mul_f64 v[12:13], v[16:17], v[4:5]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], -v[8:9], v[12:13], v[16:17]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[6:7], v[0:1], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[4:5], v[10:11], v[4:5], v[12:13]
+; GFX6-NEXT:    v_div_fixup_f64 v[2:3], v[4:5], v[2:3], 1.0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_rcp_v2f64_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX8-NEXT:    v_div_scale_f64 v[6:7], s[4:5], v[2:3], v[2:3], 1.0
+; GFX8-NEXT:    v_div_scale_f64 v[16:17], s[4:5], 1.0, v[2:3], 1.0
+; GFX8-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GFX8-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GFX8-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-NEXT:    v_div_scale_f64 v[12:13], vcc, 1.0, v[0:1], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], -v[4:5], v[8:9], 1.0
+; GFX8-NEXT:    v_fma_f64 v[18:19], -v[6:7], v[10:11], 1.0
+; GFX8-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[8:9]
+; GFX8-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX8-NEXT:    v_mul_f64 v[14:15], v[12:13], v[8:9]
+; GFX8-NEXT:    v_mul_f64 v[18:19], v[16:17], v[10:11]
+; GFX8-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[14:15], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[18:19], v[16:17]
+; GFX8-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[8:9], v[14:15]
+; GFX8-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX8-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[18:19]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
+; GFX8-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_rcp_v2f64_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[4:5], s[4:5], v[0:1], v[0:1], 1.0
+; GFX9-NEXT:    v_div_scale_f64 v[6:7], s[4:5], v[2:3], v[2:3], 1.0
+; GFX9-NEXT:    v_div_scale_f64 v[16:17], s[4:5], 1.0, v[2:3], 1.0
+; GFX9-NEXT:    v_rcp_f64_e32 v[8:9], v[4:5]
+; GFX9-NEXT:    v_rcp_f64_e32 v[10:11], v[6:7]
+; GFX9-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX9-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX9-NEXT:    v_div_scale_f64 v[12:13], vcc, 1.0, v[0:1], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], -v[4:5], v[8:9], 1.0
+; GFX9-NEXT:    v_fma_f64 v[18:19], -v[6:7], v[10:11], 1.0
+; GFX9-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[8:9]
+; GFX9-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX9-NEXT:    v_mul_f64 v[14:15], v[12:13], v[8:9]
+; GFX9-NEXT:    v_mul_f64 v[18:19], v[16:17], v[10:11]
+; GFX9-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[14:15], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[18:19], v[16:17]
+; GFX9-NEXT:    v_div_fmas_f64 v[4:5], v[4:5], v[8:9], v[14:15]
+; GFX9-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX9-NEXT:    v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[18:19]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[4:5], v[0:1], 1.0
+; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[6:7], v[2:3], 1.0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv <2 x double> <double 1.0, double 1.0>, %x, !fpmath !0
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_fdiv_v2f64_afn_ulp25(<2 x double> %a, <2 x double> %b) {
+; GCN-LABEL: v_fdiv_v2f64_afn_ulp25:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[4:5]
+; GCN-NEXT:    v_rcp_f64_e32 v[6:7], v[6:7]
+; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GCN-NEXT:    v_mul_f64 v[2:3], v[2:3], v[6:7]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn <2 x double> %a, %b, !fpmath !0
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_fdiv_v2f64_arcp_ulp25(<2 x double> %a, <2 x double> %b) {
+; GFX6-LABEL: v_fdiv_v2f64_arcp_ulp25:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX6-NEXT:    v_div_scale_f64 v[14:15], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX6-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[0:1], v[4:5], v[0:1]
+; GFX6-NEXT:    v_rcp_f64_e32 v[10:11], v[8:9]
+; GFX6-NEXT:    v_rcp_f64_e32 v[16:17], v[14:15]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v19
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v9
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], v7, v15
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[10:11], 1.0
+; GFX6-NEXT:    v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[12:13], -v[14:15], v[16:17], 1.0
+; GFX6-NEXT:    v_fma_f64 v[12:13], v[16:17], v[12:13], v[16:17]
+; GFX6-NEXT:    v_mul_f64 v[16:17], v[18:19], v[10:11]
+; GFX6-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[16:17], v[18:19]
+; GFX6-NEXT:    v_fma_f64 v[8:9], -v[14:15], v[12:13], 1.0
+; GFX6-NEXT:    v_div_fmas_f64 v[10:11], v[18:19], v[10:11], v[16:17]
+; GFX6-NEXT:    v_fma_f64 v[8:9], v[12:13], v[8:9], v[12:13]
+; GFX6-NEXT:    v_div_scale_f64 v[12:13], s[6:7], v[2:3], v[6:7], v[2:3]
+; GFX6-NEXT:    v_div_fixup_f64 v[0:1], v[10:11], v[4:5], v[0:1]
+; GFX6-NEXT:    v_mul_f64 v[16:17], v[12:13], v[8:9]
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v13
+; GFX6-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[16:17], v[12:13]
+; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GFX6-NEXT:    s_nop 1
+; GFX6-NEXT:    v_div_fmas_f64 v[8:9], v[18:19], v[8:9], v[16:17]
+; GFX6-NEXT:    v_div_fixup_f64 v[2:3], v[8:9], v[6:7], v[2:3]
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-LABEL: v_fdiv_v2f64_arcp_ulp25:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX8-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX8-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX8-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX8-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
+; GFX8-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX8-NEXT:    v_div_scale_f64 v[18:19], vcc, v[0:1], v[4:5], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX8-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[16:17], -v[10:11], v[14:15], 1.0
+; GFX8-NEXT:    v_fma_f64 v[14:15], v[14:15], v[16:17], v[14:15]
+; GFX8-NEXT:    v_mul_f64 v[16:17], v[18:19], v[12:13]
+; GFX8-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[16:17], v[18:19]
+; GFX8-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[2:3], v[6:7], v[2:3]
+; GFX8-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[12:13], v[16:17]
+; GFX8-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX8-NEXT:    v_mul_f64 v[20:21], v[18:19], v[14:15]
+; GFX8-NEXT:    v_div_fixup_f64 v[0:1], v[8:9], v[4:5], v[0:1]
+; GFX8-NEXT:    v_fma_f64 v[10:11], -v[10:11], v[20:21], v[18:19]
+; GFX8-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
+; GFX8-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_fdiv_v2f64_arcp_ulp25:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_div_scale_f64 v[8:9], s[4:5], v[4:5], v[4:5], v[0:1]
+; GFX9-NEXT:    v_div_scale_f64 v[10:11], s[4:5], v[6:7], v[6:7], v[2:3]
+; GFX9-NEXT:    v_rcp_f64_e32 v[12:13], v[8:9]
+; GFX9-NEXT:    v_rcp_f64_e32 v[14:15], v[10:11]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX9-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[14:15], 1.0
+; GFX9-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX9-NEXT:    v_div_scale_f64 v[18:19], vcc, v[0:1], v[4:5], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[8:9], v[12:13], 1.0
+; GFX9-NEXT:    v_fma_f64 v[12:13], v[12:13], v[16:17], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[16:17], -v[10:11], v[14:15], 1.0
+; GFX9-NEXT:    v_fma_f64 v[14:15], v[14:15], v[16:17], v[14:15]
+; GFX9-NEXT:    v_mul_f64 v[16:17], v[18:19], v[12:13]
+; GFX9-NEXT:    v_fma_f64 v[8:9], -v[8:9], v[16:17], v[18:19]
+; GFX9-NEXT:    v_div_scale_f64 v[18:19], s[4:5], v[2:3], v[6:7], v[2:3]
+; GFX9-NEXT:    v_div_fmas_f64 v[8:9], v[8:9], v[12:13], v[16:17]
+; GFX9-NEXT:    s_mov_b64 vcc, s[4:5]
+; GFX9-NEXT:    v_mul_f64 v[20:21], v[18:19], v[14:15]
+; GFX9-NEXT:    v_div_fixup_f64 v[0:1], v[8:9], v[4:5], v[0:1]
+; GFX9-NEXT:    v_fma_f64 v[10:11], -v[10:11], v[20:21], v[18:19]
+; GFX9-NEXT:    v_div_fmas_f64 v[10:11], v[10:11], v[14:15], v[20:21]
+; GFX9-NEXT:    v_div_fixup_f64 v[2:3], v[10:11], v[6:7], v[2:3]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv arcp <2 x double> %a, %b, !fpmath !0
+  ret <2 x double> %fdiv
+}
+
+define <2 x double> @v_fdiv_v2f64_arcp_afn_ulp25(<2 x double> %a, <2 x double> %b) {
+; GCN-LABEL: v_fdiv_v2f64_arcp_afn_ulp25:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_rcp_f64_e32 v[4:5], v[4:5]
+; GCN-NEXT:    v_rcp_f64_e32 v[6:7], v[6:7]
+; GCN-NEXT:    v_mul_f64 v[0:1], v[0:1], v[4:5]
+; GCN-NEXT:    v_mul_f64 v[2:3], v[2:3], v[6:7]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fdiv = fdiv afn arcp <2 x double> %a, %b, !fpmath !0
+  ret <2 x double> %fdiv
+}
+
+!0 = !{float 2.500000e+00}