From a94e72b5b7328ae8957e78b41b8a6f7ab093e74d Mon Sep 17 00:00:00 2001
From: Ulrich Weigand <ulrich.weigand@de.ibm.com>
Date: Thu, 20 Jan 2022 21:46:08 +0100
Subject: [PATCH] s390x: Add ISLE support

This adds ISLE support for the s390x back-end and moves lowering
of most instructions to ISLE.  The only instructions still remaining
are calls, returns, traps, and branches, most of which will need
additional support in common code.

Generated code is not intended to be (significantly) different
than before; any additional optimizations now made easier to
implement due to the ISLE layer can be added in follow-on patches.

There were a few differences in some filetests, but those are all
just simple register allocation changes (and all to the better!).
---
 cranelift/codegen/build.rs                    |    15 +
 cranelift/codegen/src/isa/s390x/inst.isle     |  2914 ++++
 cranelift/codegen/src/isa/s390x/inst/mod.rs   |  1261 +-
 cranelift/codegen/src/isa/s390x/lower.isle    |  1780 +++
 cranelift/codegen/src/isa/s390x/lower.rs      |  2241 +--
 cranelift/codegen/src/isa/s390x/lower/isle.rs |   475 +
 .../s390x/lower/isle/generated_code.manifest  |     4 +
 .../isa/s390x/lower/isle/generated_code.rs    | 11438 ++++++++++++++++
 .../filetests/filetests/isa/s390x/bitops.clif |    34 +-
 .../filetests/isa/s390x/heap_addr.clif        |     4 +-
 .../filetests/isa/s390x/shift-rotate.clif     |    28 +-
 11 files changed, 16975 insertions(+), 3219 deletions(-)
 create mode 100644 cranelift/codegen/src/isa/s390x/inst.isle
 create mode 100644 cranelift/codegen/src/isa/s390x/lower.isle
 create mode 100644 cranelift/codegen/src/isa/s390x/lower/isle.rs
 create mode 100644 cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
 create mode 100644 cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs

diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs
index 8eac58e8d9b2..cb704cb0abbb 100644
--- a/cranelift/codegen/build.rs
+++ b/cranelift/codegen/build.rs
@@ -220,6 +220,8 @@ fn get_isle_compilations(crate_dir: &std::path::Path) -> Result<IsleCompilations
         make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("isa").join("x64"));
     let src_isa_aarch64 =
         make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("isa").join("aarch64"));
+    let src_isa_s390x =
+        make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("isa").join("s390x"));
 
     // This is a set of ISLE compilation units.
     //
@@ -258,6 +260,19 @@ fn get_isle_compilations(crate_dir: &std::path::Path) -> Result<IsleCompilations
                     src_isa_aarch64.join("lower.isle"),
                 ],
             },
+            // The s390x instruction selector.
+            IsleCompilation {
+                output: src_isa_s390x
+                    .join("lower")
+                    .join("isle")
+                    .join("generated_code.rs"),
+                inputs: vec![
+                    clif_isle.clone(),
+                    prelude_isle.clone(),
+                    src_isa_s390x.join("inst.isle"),
+                    src_isa_s390x.join("lower.isle"),
+                ],
+            },
         ],
     })
 }
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
new file mode 100644
index 000000000000..591671da6655
--- /dev/null
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -0,0 +1,2914 @@
+;; Instruction formats.
+(type MInst
+  (enum
+    ;; A no-op of zero size.
+    (Nop0)
+
+    ;; A no-op of size two bytes.
+    (Nop2)
+
+    ;; An ALU operation with two register sources and a register destination.
+    (AluRRR
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (rn Reg)
+      (rm Reg))
+
+    ;; An ALU operation with a register source and a signed 16-bit
+    ;; immediate source, and a separate register destination.
+    (AluRRSImm16
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (rn Reg)
+      (imm i16))
+
+    ;; An ALU operation with a register in-/out operand and
+    ;; a second register source.
+    (AluRR
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (rm Reg))
+
+    ;; An ALU operation with a register in-/out operand and
+    ;; a memory source.
+    (AluRX
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; An ALU operation with a register in-/out operand and a signed 16-bit
+    ;; immediate source.
+    (AluRSImm16
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (imm i16))
+
+    ;; An ALU operation with a register in-/out operand and a signed 32-bit
+    ;; immediate source.
+    (AluRSImm32
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (imm i32))
+
+    ;; An ALU operation with a register in-/out operand and an unsigned 32-bit
+    ;; immediate source.
+    (AluRUImm32
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (imm u32))
+
+    ;; An ALU operation with a register in-/out operand and a shifted 16-bit
+    ;; immediate source.
+    (AluRUImm16Shifted
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (imm UImm16Shifted))
+
+    ;; An ALU operation with a register in-/out operand and a shifted 32-bit
+    ;; immediate source.
+    (AluRUImm32Shifted
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (imm UImm32Shifted))
+
+    ;; A multiply operation with two register sources and a register pair destination.
+    ;; FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
+    (SMulWide
+      (rn Reg)
+      (rm Reg))
+
+    ;; A multiply operation with an in/out register pair, and an extra register source.
+    ;; Only the lower half of the register pair is used as input.
+    ;; FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
+    (UMulWide
+      (rn Reg))
+
+    ;; A divide operation with an in/out register pair, and an extra register source.
+    ;; Only the lower half of the register pair is used as input.
+    ;; FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
+    (SDivMod32
+      (rn Reg))
+    (SDivMod64
+      (rn Reg))
+
+    ;; A divide operation with an in/out register pair, and an extra register source.
+    ;; FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
+    (UDivMod32
+      (rn Reg))
+    (UDivMod64
+      (rn Reg))
+
+    ;; A FLOGR operation with a register source and a register pair destination.
+    ;; FIXME The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
+    (Flogr
+      (rn Reg))
+
+    ;; A shift instruction with a register source, a register destination,
+    ;; and an immediate plus an optional register as shift count.
+    (ShiftRR
+      (shift_op ShiftOp)
+      (rd WritableReg)
+      (rn Reg)
+      (shift_imm u8)
+      (shift_reg Reg))
+
+    ;; An unary operation with a register source and a register destination.
+    (UnaryRR
+      (op UnaryOp)
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; A compare operation with two register sources.
+    (CmpRR
+      (op CmpOp)
+      (rn Reg)
+      (rm Reg))
+
+    ;; A compare operation with a register source and a memory source.
+    (CmpRX
+      (op CmpOp)
+      (rn Reg)
+      (mem MemArg))
+
+    ;; A compare operation with a register source and a signed 16-bit
+    ;; immediate source.
+    (CmpRSImm16
+      (op CmpOp)
+      (rn Reg)
+      (imm i16))
+
+    ;; A compare operation with a register source and a signed 32-bit
+    ;; immediate source.
+    (CmpRSImm32
+      (op CmpOp)
+      (rn Reg)
+      (imm i32))
+
+    ;; A compare operation with a register source and a unsigned 32-bit
+    ;; immediate source.
+    (CmpRUImm32
+      (op CmpOp)
+      (rn Reg)
+      (imm u32))
+
+    ;; A compare-and-trap instruction with two register sources.
+    (CmpTrapRR
+      (op CmpOp)
+      (rn Reg)
+      (rm Reg)
+      (cond Cond)
+      (trap_code TrapCode))
+
+    ;; A compare-and-trap operation with a register source and a signed 16-bit
+    ;; immediate source.
+    (CmpTrapRSImm16
+      (op CmpOp)
+      (rn Reg)
+      (imm i16)
+      (cond Cond)
+      (trap_code TrapCode))
+
+    ;; A compare-and-trap operation with a register source and an unsigned 16-bit
+    ;; immediate source.
+    (CmpTrapRUImm16
+      (op CmpOp)
+      (rn Reg)
+      (imm u16)
+      (cond Cond)
+      (trap_code TrapCode))
+
+    ;; An atomic read-modify-write operation with a memory in-/out operand,
+    ;; a register destination, and a register source.
+    ;; a memory source.
+    (AtomicRmw
+      (alu_op ALUOp)
+      (rd WritableReg)
+      (rn Reg)
+      (mem MemArg))
+
+    ;; A 32-bit atomic compare-and-swap operation.
+    (AtomicCas32
+      (rd WritableReg)
+      (rn Reg)
+      (mem MemArg))
+
+    ;; A 64-bit atomic compare-and-swap operation.
+    (AtomicCas64
+      (rd WritableReg)
+      (rn Reg)
+      (mem MemArg))
+
+    ;; A memory fence operation.
+    (Fence)
+
+    ;; A 32-bit load.
+    (Load32
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; An unsigned (zero-extending) 8-bit to 32-bit load.
+    (Load32ZExt8
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A signed (sign-extending) 8-bit to 32-bit load.
+    (Load32SExt8
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; An unsigned (zero-extending) 16-bit to 32-bit load.
+    (Load32ZExt16
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A signed (sign-extending) 16-bit to 32-bit load.
+    (Load32SExt16
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A 64-bit load.
+    (Load64
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; An unsigned (zero-extending) 8-bit to 64-bit load.
+    (Load64ZExt8
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A signed (sign-extending) 8-bit to 64-bit load.
+    (Load64SExt8
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; An unsigned (zero-extending) 16-bit to 64-bit load.
+    (Load64ZExt16
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A signed (sign-extending) 16-bit to 64-bit load.
+    (Load64SExt16
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; An unsigned (zero-extending) 32-bit to 64-bit load.
+    (Load64ZExt32
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A signed (sign-extending) 32-bit to 64-bit load.
+    (Load64SExt32
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A 16-bit byte-reversed load.
+    (LoadRev16
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A 32-bit byte-reversed load.
+    (LoadRev32
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; A 64-bit byte-reversed load.
+    (LoadRev64
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; An 8-bit store.
+    (Store8
+      (rd Reg)
+      (mem MemArg))
+
+    ;; A 16-bit store.
+    (Store16
+      (rd Reg)
+      (mem MemArg))
+
+    ;; A 32-bit store.
+    (Store32
+      (rd Reg)
+      (mem MemArg))
+
+    ;; A 64-bit store.
+    (Store64
+      (rd Reg)
+      (mem MemArg))
+
+    ;; An 8-bit store of an immediate.
+    (StoreImm8
+      (imm u8)
+      (mem MemArg))
+
+    ;; A 16-bit store of an immediate.
+    (StoreImm16
+      (imm i16)
+      (mem MemArg))
+
+    ;; A 32-bit store of a sign-extended 16-bit immediate.
+    (StoreImm32SExt16
+      (imm i16)
+      (mem MemArg))
+
+    ;; A 64-bit store of a sign-extended 16-bit immediate.
+    (StoreImm64SExt16
+      (imm i16)
+      (mem MemArg))
+
+    ;; A 16-bit byte-reversed store.
+    (StoreRev16
+      (rd Reg)
+      (mem MemArg))
+
+    ;; A 32-bit byte-reversed store.
+    (StoreRev32
+      (rd Reg)
+      (mem MemArg))
+
+    ;; A 64-bit byte-reversed store.
+    (StoreRev64
+      (rd Reg)
+      (mem MemArg))
+
+    ;; A load-multiple instruction.
+    (LoadMultiple64
+      (rt WritableReg)
+      (rt2 WritableReg)
+      (mem MemArg))
+
+    ;; A store-multiple instruction.
+    (StoreMultiple64
+      (rt Reg)
+      (rt2 Reg)
+      (mem MemArg))
+
+    ;; A 32-bit move instruction.
+    (Mov32
+      (rd WritableReg)
+      (rm Reg))
+
+    ;; A 64-bit move instruction.
+    (Mov64
+      (rd WritableReg)
+      (rm Reg))
+
+    ;; A 32-bit move instruction with a full 32-bit immediate.
+    (Mov32Imm
+      (rd WritableReg)
+      (imm u32))
+
+    ;; A 32-bit move instruction with a 16-bit signed immediate.
+    (Mov32SImm16
+      (rd WritableReg)
+      (imm i16))
+
+    ;; A 64-bit move instruction with a 16-bit signed immediate.
+    (Mov64SImm16
+      (rd WritableReg)
+      (imm i16))
+
+    ;; A 64-bit move instruction with a 32-bit signed immediate.
+    (Mov64SImm32
+      (rd WritableReg)
+      (imm i32))
+
+    ;; A 64-bit move instruction with a shifted 16-bit immediate.
+    (Mov64UImm16Shifted
+      (rd WritableReg)
+      (imm UImm16Shifted))
+
+    ;; A 64-bit move instruction with a shifted 32-bit immediate.
+    (Mov64UImm32Shifted
+      (rd WritableReg)
+      (imm UImm32Shifted))
+
+    ;; A 64-bit insert instruction with a shifted 16-bit immediate.
+    (Insert64UImm16Shifted
+      (rd WritableReg)
+      (imm UImm16Shifted))
+
+    ;; A 64-bit insert instruction with a shifted 32-bit immediate.
+    (Insert64UImm32Shifted
+      (rd WritableReg)
+      (imm UImm32Shifted))
+
+    ;; A sign- or zero-extend operation.
+    (Extend
+      (rd WritableReg)
+      (rn Reg)
+      (signed bool)
+      (from_bits u8)
+      (to_bits u8))
+
+    ;; A 32-bit conditional move instruction.
+    (CMov32
+      (rd WritableReg)
+      (cond Cond)
+      (rm Reg))
+
+    ;; A 64-bit conditional move instruction.
+    (CMov64
+      (rd WritableReg)
+      (cond Cond)
+      (rm Reg))
+
+    ;; A 32-bit conditional move instruction with a 16-bit signed immediate.
+    (CMov32SImm16
+      (rd WritableReg)
+      (cond Cond)
+      (imm i16))
+
+    ;; A 64-bit conditional move instruction with a 16-bit signed immediate.
+    (CMov64SImm16
+      (rd WritableReg)
+      (cond Cond)
+      (imm i16))
+
+    ;; A 32-bit FPU move.
+    (FpuMove32
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; A 64-bit FPU move.
+    (FpuMove64
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; A 32-bit conditional move FPU instruction.
+    (FpuCMov32
+      (rd WritableReg)
+      (cond Cond)
+      (rm Reg))
+
+    ;; A 64-bit conditional move FPU instruction.
+    (FpuCMov64
+      (rd WritableReg)
+      (cond Cond)
+      (rm Reg))
+
+    ;; A 64-bit move instruction from GPR to FPR.
+    (MovToFpr
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; A 64-bit move instruction from FPR to GPR.
+    (MovFromFpr
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; 1-op FPU instruction.
+    (FpuRR
+      (fpu_op FPUOp1)
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; 2-op FPU instruction.
+    (FpuRRR
+      (fpu_op FPUOp2)
+      (rd WritableReg)
+      (rm Reg))
+
+    ;; 3-op FPU instruction.
+    (FpuRRRR
+      (fpu_op FPUOp3)
+      (rd WritableReg)
+      (rn Reg)
+      (rm Reg))
+
+    ;; FPU copy sign instruction.
+    (FpuCopysign
+      (rd WritableReg)
+      (rn Reg)
+      (rm Reg))
+
+    ;; FPU comparison, single-precision (32 bit).
+    (FpuCmp32
+      (rn Reg)
+      (rm Reg))
+
+    ;; FPU comparison, double-precision (64 bit).
+    (FpuCmp64
+      (rn Reg)
+      (rm Reg))
+
+    ;; Floating-point load, single-precision (32 bit).
+    (FpuLoad32
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; Floating-point store, single-precision (32 bit).
+    (FpuStore32
+      (rd Reg)
+      (mem MemArg))
+
+    ;; Floating-point load, double-precision (64 bit).
+    (FpuLoad64
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; Floating-point store, double-precision (64 bit).
+    (FpuStore64
+      (rd Reg)
+      (mem MemArg))
+
+    ;; Floating-point byte-reversed load, single-precision (32 bit).
+    (FpuLoadRev32
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; Floating-point byte-reversed store, single-precision (32 bit).
+    (FpuStoreRev32
+      (rd Reg)
+      (mem MemArg))
+
+    ;; Floating-point byte-reversed load, double-precision (64 bit).
+    (FpuLoadRev64
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; Floating-point byte-reversed store, double-precision (64 bit).
+    (FpuStoreRev64
+      (rd Reg)
+      (mem MemArg))
+
+    ;; Load floating-point constant, single-precision (32 bit).
+    (LoadFpuConst32
+      (rd WritableReg)
+      (const_data u32))
+
+    ;; Load floating-point constant, double-precision (64 bit).
+    (LoadFpuConst64
+      (rd WritableReg)
+      (const_data u64))
+
+    ;; Conversion FP -> integer.
+    (FpuToInt
+      (op FpuToIntOp)
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; Conversion integer -> FP.
+    (IntToFpu
+      (op IntToFpuOp)
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; Round to integer.
+    (FpuRound
+      (op FpuRoundMode)
+      (rd WritableReg)
+      (rn Reg))
+
+    ;; 2-op FPU instruction implemented as vector instruction with the W bit.
+    (FpuVecRRR
+      (fpu_op FPUOp2)
+      (rd WritableReg)
+      (rn Reg)
+      (rm Reg))
+
+    ;; A machine call instruction.
+    (Call
+      (link WritableReg)
+      (info BoxCallInfo))
+
+    ;; A machine indirect-call instruction.
+    (CallInd
+      (link WritableReg)
+      (info BoxCallIndInfo))
+
+    ;; ---- branches (exactly one must appear at end of BB) ----
+
+    ;; A machine return instruction.
+    (Ret
+      (link Reg))
+
+    ;; A placeholder instruction, generating no code, meaning that a function epilogue must be
+    ;; inserted there.
+    (EpiloguePlaceholder)
+
+    ;; An unconditional branch.
+    (Jump
+      (dest BranchTarget))
+
+    ;; A conditional branch. Contains two targets; at emission time, both are emitted, but
+    ;; the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the
+    ;; choice of taken/not_taken (inverting the branch polarity as needed) based on the
+    ;; fallthrough at the time of lowering.
+    (CondBr
+      (taken BranchTarget)
+      (not_taken BranchTarget)
+      (cond Cond))
+
+    ;; A conditional trap execute a `Trap` if the condition is true. This is
+    ;; one VCode instruction because it uses embedded control flow; it is
+    ;; logically a single-in, single-out region, but needs to appear as one
+    ;; unit to the register allocator.
+    ;;
+    ;; The `Cond` gives the conditional-branch condition that will
+    ;; *execute* the embedded `Trap`. (In the emitted code, we use the inverse
+    ;; of this condition in a branch that skips the trap instruction.)
+    (TrapIf
+      (cond Cond)
+      (trap_code TrapCode))
+
+    ;; A one-way conditional branch, invisible to the CFG processing; used *only* as part of
+    ;; straight-line sequences in code to be emitted.
+    ;;
+    ;; In more detail:
+    ;; - This branch is lowered to a branch at the machine-code level, but does not end a basic
+    ;;   block, and does not create edges in the CFG seen by regalloc.
+    ;; - Thus, it is *only* valid to use as part of a single-in, single-out sequence that is
+    ;;   lowered from a single CLIF instruction. For example, certain arithmetic operations may
+    ;;   use these branches to handle certain conditions, such as overflows, traps, etc.
+    ;;
+    ;; See, e.g., the lowering of `trapif` (conditional trap) for an example.
+    (OneWayCondBr
+      (target BranchTarget)
+      (cond Cond))
+
+    ;; An indirect branch through a register, augmented with set of all
+    ;; possible successors.
+    (IndirectBr
+      (rn Reg)
+      (targets VecMachLabel))
+
+    ;; A "debugtrap" instruction, used for e.g. traps and debug breakpoints.
+    (Debugtrap)
+
+    ;; An instruction guaranteed to always be undefined and to trigger an illegal instruction at
+    ;; runtime.
+    (Trap
+      (trap_code TrapCode))
+
+    ;; Jump-table sequence, as one compound instruction (see note in lower.rs
+    ;; for rationale).
+    (JTSequence
+      (info BoxJTSequenceInfo)
+      (ridx Reg)
+      (rtmp1 WritableReg)
+      (rtmp2 WritableReg))
+
+    ;; Load an inline symbol reference with RelocDistance::Far.
+    (LoadExtNameFar
+      (rd WritableReg)
+      (name BoxExternalName)
+      (offset i64))
+
+    ;; Load address referenced by `mem` into `rd`.
+    (LoadAddr
+      (rd WritableReg)
+      (mem MemArg))
+
+    ;; Marker, no-op in generated code SP "virtual offset" is adjusted. This
+    ;; controls how MemArg::NominalSPOffset args are lowered.
+    (VirtualSPOffsetAdj
+      (offset i64))
+
+    ;; A definition of a value label.
+    (ValueLabelMarker
+      (reg Reg)
+      (label ValueLabel))
+
+    ;; An unwind pseudoinstruction describing the state of the
+    ;; machine at this program point.
+    (Unwind
+      (inst UnwindInst))
+))
+
+;; Primitive types used in instruction formats.
+
+(type BoxCallInfo (primitive BoxCallInfo))
+(type BoxCallIndInfo (primitive BoxCallIndInfo))
+(type VecMachLabel (primitive VecMachLabel))
+(type BranchTarget (primitive BranchTarget))
+(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
+(type BoxExternalName (primitive BoxExternalName))
+(type ValueLabel (primitive ValueLabel))
+(type UnwindInst (primitive UnwindInst))
+
+;; An ALU operation.
+(type ALUOp
+  (enum
+    (Add32)
+    (Add32Ext16)
+    (Add64)
+    (Add64Ext16)
+    (Add64Ext32)
+    (AddLogical32)
+    (AddLogical64)
+    (AddLogical64Ext32)
+    (Sub32)
+    (Sub32Ext16)
+    (Sub64)
+    (Sub64Ext16)
+    (Sub64Ext32)
+    (SubLogical32)
+    (SubLogical64)
+    (SubLogical64Ext32)
+    (Mul32)
+    (Mul32Ext16)
+    (Mul64)
+    (Mul64Ext16)
+    (Mul64Ext32)
+    (And32)
+    (And64)
+    (Orr32)
+    (Orr64)
+    (Xor32)
+    (Xor64)
+    ;; NAND
+    (AndNot32)
+    (AndNot64)
+    ;; NOR
+    (OrrNot32)
+    (OrrNot64)
+    ;; XNOR
+    (XorNot32)
+    (XorNot64)
+))
+
+;; A unary operation.
+(type UnaryOp
+  (enum
+    (Abs32)
+    (Abs64)
+    (Abs64Ext32)
+    (Neg32)
+    (Neg64)
+    (Neg64Ext32)
+    (PopcntByte)
+    (PopcntReg)
+))
+
+;; A shift operation.
+(type ShiftOp
+  (enum
+    (RotL32)
+    (RotL64)
+    (LShL32)
+    (LShL64)
+    (LShR32)
+    (LShR64)
+    (AShR32)
+    (AShR64)
+))
+
+;; An integer comparison operation.
+(type CmpOp
+  (enum
+    (CmpS32)
+    (CmpS32Ext16)
+    (CmpS64)
+    (CmpS64Ext16)
+    (CmpS64Ext32)
+    (CmpL32)
+    (CmpL32Ext16)
+    (CmpL64)
+    (CmpL64Ext16)
+    (CmpL64Ext32)
+))
+
+;; A floating-point unit (FPU) operation with one arg.
+(type FPUOp1
+  (enum
+    (Abs32)
+    (Abs64)
+    (Neg32)
+    (Neg64)
+    (NegAbs32)
+    (NegAbs64)
+    (Sqrt32)
+    (Sqrt64)
+    (Cvt32To64)
+    (Cvt64To32)
+))
+
+;; A floating-point unit (FPU) operation with two args.
+(type FPUOp2
+  (enum
+    (Add32)
+    (Add64)
+    (Sub32)
+    (Sub64)
+    (Mul32)
+    (Mul64)
+    (Div32)
+    (Div64)
+    (Max32)
+    (Max64)
+    (Min32)
+    (Min64)
+))
+
+;; A floating-point unit (FPU) operation with three args.
+(type FPUOp3
+  (enum
+    (MAdd32)
+    (MAdd64)
+    (MSub32)
+    (MSub64)
+))
+
+;; A conversion from an FP to an integer value.
+(type FpuToIntOp
+  (enum
+    (F32ToU32)
+    (F32ToI32)
+    (F32ToU64)
+    (F32ToI64)
+    (F64ToU32)
+    (F64ToI32)
+    (F64ToU64)
+    (F64ToI64)
+))
+
+;; A conversion from an integer to an FP value.
+(type IntToFpuOp
+  (enum
+    (U32ToF32)
+    (I32ToF32)
+    (U32ToF64)
+    (I32ToF64)
+    (U64ToF32)
+    (I64ToF32)
+    (U64ToF64)
+    (I64ToF64)
+))
+
+;; Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero
+;; (trunc), or to nearest, and for 32- or 64-bit FP values.
+(type FpuRoundMode
+  (enum
+    (Minus32)
+    (Minus64)
+    (Plus32)
+    (Plus64)
+    (Zero32)
+    (Zero64)
+    (Nearest32)
+    (Nearest64)
+))
+
+
+;; Helpers for querying enabled ISA extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl mie2_enabled () Type)
+(extern extractor mie2_enabled mie2_enabled)
+(decl mie2_disabled () Type)
+(extern extractor mie2_disabled mie2_disabled)
+
+(decl vxrs_ext2_enabled () Type)
+(extern extractor vxrs_ext2_enabled vxrs_ext2_enabled)
+(decl vxrs_ext2_disabled () Type)
+(extern extractor vxrs_ext2_disabled vxrs_ext2_disabled)
+
+(decl allow_div_traps () Type)
+(extern extractor allow_div_traps allow_div_traps)
+
+
+;; Helpers to access instruction data members ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Extractor for `symbol_value` instruction data member.
+
+(decl symbol_value_data (BoxExternalName RelocDistance i64) Inst)
+(extern extractor symbol_value_data symbol_value_data)
+
+;; Extractor for `call_target` instruction data members.
+
+(decl call_target_data (BoxExternalName RelocDistance) Inst)
+(extern extractor call_target_data call_target_data)
+
+
+;; Helpers for register numbers and types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Hard-coded registers.
+(decl writable_gpr (u8) WritableReg)
+(extern constructor writable_gpr writable_gpr)
+
+;; The zero register.
+(decl zero_reg () Reg)
+(extern constructor zero_reg zero_reg)
+
+;; Types that can be operated on using 32-bit GPR instructions
+(decl gpr32_ty (Type) Type)
+(extern extractor gpr32_ty gpr32_ty)
+
+;; Types that can be operated on using 64-bit GPR instructions
+(decl gpr64_ty (Type) Type)
+(extern extractor gpr64_ty gpr64_ty)
+
+
+;; Helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Special integer types and their constructors
+
+(type UImm32Shifted (primitive UImm32Shifted))
+(decl uimm32shifted (u32 u8) UImm32Shifted)
+(extern constructor uimm32shifted uimm32shifted)
+
+(type UImm16Shifted (primitive UImm16Shifted))
+(decl uimm16shifted (u16 u8) UImm16Shifted)
+(extern constructor uimm16shifted uimm16shifted)
+
+;; Detect specific integer values
+
+(decl i64_nonequal (i64 i64) i64)
+(extern extractor i64_nonequal i64_nonequal (out in))
+
+(decl i64_nonzero (i64) i64)
+(extractor (i64_nonzero val) (i64_nonequal val <0))
+
+(decl i64_not_neg1 (i64) i64)
+(extractor (i64_not_neg1 val) (i64_nonequal val <-1))
+
+;; Integer type casts (with the rust `as` semantics).
+
+(decl u8_as_u16 (u8) u16)
+(extern constructor u8_as_u16 u8_as_u16)
+
+(decl u64_as_u32 (u64) u32)
+(extern constructor u64_as_u32 u64_as_u32)
+
+(decl u64_as_i16 (u64) i16)
+(extern constructor u64_as_i16 u64_as_i16)
+
+;; Split an u64 into high and low parts.
+
+(decl u64_nonzero_hipart (u64) u64)
+(extern extractor u64_nonzero_hipart u64_nonzero_hipart)
+
+(decl u64_nonzero_lopart (u64) u64)
+(extern extractor u64_nonzero_lopart u64_nonzero_lopart)
+
+;; Extract smaller integer type from u64 if it matches.
+
+(decl i32_from_u64 (i32) u64)
+(extern extractor i32_from_u64 i32_from_u64)
+
+(decl i16_from_u64 (i16) u64)
+(extern extractor i16_from_u64 i16_from_u64)
+
+(decl uimm32shifted_from_u64 (UImm32Shifted) u64)
+(extern extractor uimm32shifted_from_u64 uimm32shifted_from_u64)
+
+(decl uimm16shifted_from_u64 (UImm16Shifted) u64)
+(extern extractor uimm16shifted_from_u64 uimm16shifted_from_u64)
+
+;; Extract integer of certain type from value if it matches.
+
+(decl u64_from_value (u64) Value)
+(extern extractor u64_from_value u64_from_value)
+
+(decl u32_from_value (u32) Value)
+(extern extractor u32_from_value u32_from_value)
+
+(decl u8_from_value (u8) Value)
+(extern extractor u8_from_value u8_from_value)
+
+(decl u64_from_signed_value (u64) Value)
+(extern extractor u64_from_signed_value u64_from_signed_value)
+
+(decl i64_from_value (i64) Value)
+(extern extractor i64_from_value i64_from_value)
+
+(decl i32_from_value (i32) Value)
+(extern extractor i32_from_value i32_from_value)
+
+(decl i16_from_value (i16) Value)
+(extern extractor i16_from_value i16_from_value)
+
+(decl i16_from_swapped_value (i16) Value)
+(extern extractor i16_from_swapped_value i16_from_swapped_value)
+
+(decl i64_from_negated_value (i64) Value)
+(extern extractor i64_from_negated_value i64_from_negated_value)
+
+(decl i32_from_negated_value (i32) Value)
+(extern extractor i32_from_negated_value i32_from_negated_value)
+
+(decl i16_from_negated_value (i16) Value)
+(extern extractor i16_from_negated_value i16_from_negated_value)
+
+(decl uimm16shifted_from_value (UImm16Shifted) Value)
+(extern extractor uimm16shifted_from_value uimm16shifted_from_value)
+
+(decl uimm32shifted_from_value (UImm32Shifted) Value)
+(extern extractor uimm32shifted_from_value uimm32shifted_from_value)
+
+(decl uimm16shifted_from_inverted_value (UImm16Shifted) Value)
+(extern extractor uimm16shifted_from_inverted_value uimm16shifted_from_inverted_value)
+
+(decl uimm32shifted_from_inverted_value (UImm32Shifted) Value)
+(extern extractor uimm32shifted_from_inverted_value uimm32shifted_from_inverted_value)
+
+
+;; Helpers for masking shift amounts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Mask (immediate) shift amount to the type size.
+(decl mask_amt_imm (Type i64) u8)
+(extern constructor mask_amt_imm mask_amt_imm)
+
+;; Mask (immediate) shift amount to the type size.
+;; Note that the hardware instructions always masks to six bits, so
+;; in the case of a 64-bit type we do not need any explicit masking.
+(decl mask_amt_reg (Type Reg) Reg)
+(rule (mask_amt_reg (gpr32_ty ty) reg)
+      (let ((mask u8 (mask_amt_imm ty -1)))
+        (and_uimm16shifted ty reg (uimm16shifted (u8_as_u16 mask) 0))))
+(rule (mask_amt_reg (gpr64_ty ty) reg) reg)
+
+
+;; Helpers for condition codes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(type Cond extern (enum))
+
+(decl mask_as_cond (u8) Cond)
+(extern constructor mask_as_cond mask_as_cond)
+
+(decl intcc_as_cond (IntCC) Cond)
+(extern constructor intcc_as_cond intcc_as_cond)
+
+(decl floatcc_as_cond (FloatCC) Cond)
+(extern constructor floatcc_as_cond floatcc_as_cond)
+
+(decl invert_cond (Cond) Cond)
+(extern constructor invert_cond invert_cond)
+
+(decl signed () IntCC)
+(extern extractor signed signed)
+
+(decl unsigned () IntCC)
+(extern extractor unsigned unsigned)
+
+
+;; Helpers for memory arguments ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Accessors for `RelocDistance`.
+
+(type RelocDistance extern (enum))
+
+(decl reloc_distance_near () RelocDistance)
+(extern extractor reloc_distance_near reloc_distance_near)
+
+;; Accessors for `Offset32`.
+
+(decl zero_offset () Offset32)
+(extern constructor zero_offset zero_offset)
+
+(decl i64_from_offset (i64) Offset32)
+(extern extractor infallible i64_from_offset i64_from_offset)
+
+;; Accessors for `MemFlags`.
+
+(decl littleendian () MemFlags)
+(extern extractor littleendian littleendian)
+
+(decl bigendian () MemFlags)
+(extern extractor bigendian bigendian)
+
+(decl memflags_trusted () MemFlags)
+(extern constructor memflags_trusted memflags_trusted)
+
+;; Accessors for `MemArg`.
+
+(type MemArg extern (enum))
+
+(decl memarg_reg_plus_reg (Reg Reg MemFlags) MemArg)
+(extern constructor memarg_reg_plus_reg memarg_reg_plus_reg)
+
+(decl memarg_reg_plus_off (Reg i64 MemFlags) MemArg)
+(extern constructor memarg_reg_plus_off memarg_reg_plus_off)
+
+(decl memarg_symbol (BoxExternalName i32 MemFlags) MemArg)
+(extern constructor memarg_symbol memarg_symbol)
+
+;; Form the sum of two offset values, and check that the result is
+;; a valid `MemArg::Symbol` offset (i.e. is even and fits into i32).
+(decl memarg_symbol_offset_sum (i64 i32) i64)
+(extern extractor memarg_symbol_offset_sum memarg_symbol_offset_sum (in out))
+
+;; Likewise, but just check a single offset value.
+(decl memarg_symbol_offset (i32) i64)
+(extractor (memarg_symbol_offset offset) (memarg_symbol_offset_sum <0 offset))
+
+;; Lower an address into a `MemArg`.
+
+(decl lower_address (MemFlags Value Offset32) MemArg)
+
+(rule (lower_address flags addr (i64_from_offset offset))
+      (memarg_reg_plus_off (put_in_reg addr) offset flags))
+
+(rule (lower_address flags (def_inst (iadd x y)) (i64_from_offset 0))
+      (memarg_reg_plus_reg (put_in_reg x) (put_in_reg y) flags))
+
+(rule (lower_address flags
+        (def_inst (symbol_value_data name (reloc_distance_near) offset))
+        (i64_from_offset (memarg_symbol_offset_sum <offset final_offset)))
+      (memarg_symbol name final_offset flags))
+
+
+;; Test whether a `load` address will be lowered to a `MemArg::Symbol`.
+
+(decl load_sym (Inst) Inst)
+(extractor (load_sym inst)
+  (and inst
+       (load _ (def_inst (symbol_value_data _ (reloc_distance_near) offset))
+               (i64_from_offset (memarg_symbol_offset_sum <offset _)))))
+
+(decl uload16_sym (Inst) Inst)
+(extractor (uload16_sym inst)
+  (and inst
+       (uload16 _ (def_inst (symbol_value_data _ (reloc_distance_near) offset))
+                  (i64_from_offset (memarg_symbol_offset_sum <offset _)))))
+
+
+;; Helpers for stack-slot addresses ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl abi_stackslot_addr (WritableReg StackSlot Offset32) MInst)
+(extern constructor abi_stackslot_addr abi_stackslot_addr)
+
+(decl stack_addr_impl (Type StackSlot Offset32) Reg)
+(rule (stack_addr_impl ty stack_slot offset)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (abi_stackslot_addr dst stack_slot offset))))
+        (writable_reg_to_reg dst)))
+
+
+;; Helpers for extracting extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; A value that is the result of a sign-extend from a 32-bit value.
+(decl sext32_value (Value) Value)
+(extractor (sext32_value x) (def_inst (sextend (and x (value_type $I32)))))
+
+;; A value that is the result of a zero-extend from a 32-bit value.
+(decl zext32_value (Value) Value)
+(extractor (zext32_value x) (def_inst (uextend (and x (value_type $I32)))))
+
+
+;; Helpers for sinkable loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Extract a sinkable instruction from a value operand.
+(decl sinkable_inst (Inst) Value)
+(extern extractor sinkable_inst sinkable_inst)
+
+;; Sink a sinkable instruction.
+;;
+;; This is a side-effectful operation that notifies the context that the
+;; sinkable instruction been sunk into another instruction, and no longer
+;; needs to be lowered.
+(decl sink_inst (Inst) Unit)
+(extern constructor sink_inst sink_inst)
+
+
+;; Sinkable big-endian load instruction.
+(decl sinkable_load (Inst) Value)
+(extractor (sinkable_load inst)
+           (sinkable_inst (and inst (load (bigendian) _addr _offset))))
+
+;; Sinkable big-endian load instruction (32/64-bit types only).
+(decl sinkable_load_32_64 (Inst) Value)
+(extractor (sinkable_load_32_64 inst)
+           (and (value_type (ty_32_or_64 _)) (sinkable_load inst)))
+
+;; Sinkable big-endian load instruction (16-bit types only).
+(decl sinkable_load_16 (Inst) Value)
+(extractor (sinkable_load_16 inst)
+           (and (value_type $I16) (sinkable_load inst)))
+
+;; Sinkable big-endian sload16 instruction.
+(decl sinkable_sload16 (Inst) Value)
+(extractor (sinkable_sload16 inst)
+           (sinkable_inst (and inst (sload16 (bigendian) _addr _offset))))
+
+;; Sinkable big-endian sload32 instruction.
+(decl sinkable_sload32 (Inst) Value)
+(extractor (sinkable_sload32 inst)
+           (sinkable_inst (and inst (sload32 (bigendian) _addr _offset))))
+
+;; Sinkable big-endian uload16 instruction.
+(decl sinkable_uload16 (Inst) Value)
+(extractor (sinkable_uload16 inst)
+           (sinkable_inst (and inst (uload16 (bigendian) _addr _offset))))
+
+;; Sinkable big-endian uload32 instruction.
+(decl sinkable_uload32 (Inst) Value)
+(extractor (sinkable_uload32 inst)
+           (sinkable_inst (and inst (uload32 (bigendian) _addr _offset))))
+
+;; Sink a load instruction, returning its address as `MemArg`.
+;; This is a side-effectful operation, see `sink_inst`.
+(decl sink_load (Inst) MemArg)
+(rule (sink_load inst @ (load flags addr offset))
+      (let ((_ Unit (sink_inst inst)))
+        (lower_address flags addr offset)))
+
+;; Sink a sload16 instruction, returning its address as `MemArg`.
+;; This is a side-effectful operation, see `sink_inst`.
+(decl sink_sload16 (Inst) MemArg)
+(rule (sink_sload16 inst @ (sload16 flags addr offset))
+      (let ((_ Unit (sink_inst inst)))
+        (lower_address flags addr offset)))
+
+;; Sink a sload32 instruction, returning its address as `MemArg`.
+;; This is a side-effectful operation, see `sink_inst`.
+(decl sink_sload32 (Inst) MemArg)
+(rule (sink_sload32 inst @ (sload32 flags addr offset))
+      (let ((_ Unit (sink_inst inst)))
+        (lower_address flags addr offset)))
+
+;; Sink a uload16 instruction, returning its address as `MemArg`.
+;; This is a side-effectful operation, see `sink_inst`.
+(decl sink_uload16 (Inst) MemArg)
+(rule (sink_uload16 inst @ (uload16 flags addr offset))
+      (let ((_ Unit (sink_inst inst)))
+        (lower_address flags addr offset)))
+
+;; Sink a uload32 instruction, returning its address as `MemArg`.
+;; This is a side-effectful operation, see `sink_inst`.
+(decl sink_uload32 (Inst) MemArg)
+(rule (sink_uload32 inst @ (uload32 flags addr offset))
+      (let ((_ Unit (sink_inst inst)))
+        (lower_address flags addr offset)))
+
+
+;; Helpers for register pairs ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; A writable register pair.
+(type WritableRegPair (enum (WritableRegPair (hi WritableReg) (lo WritableReg))))
+
+;; Allocate a writable register pair.
+;; FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
+(decl temp_writable_regpair () WritableRegPair)
+(rule (temp_writable_regpair)
+      (WritableRegPair.WritableRegPair (writable_gpr 0) (writable_gpr 1)))
+
+;; Allocate a writable register pair and initialize it as a copy of the input.
+;; FIXME: Because there is only a single hard-coded regpair, the copy is a no-op.
+(decl copy_writable_regpair (RegPair) WritableRegPair)
+(rule (copy_writable_regpair _src) (temp_writable_regpair))
+
+;; Retrieve the high word of the writable register pair.
+(decl writable_regpair_hi (WritableRegPair) WritableReg)
+(rule (writable_regpair_hi (WritableRegPair.WritableRegPair hi _)) hi)
+
+;; Retrieve the low word of the writable register pair.
+(decl writable_regpair_lo (WritableRegPair) WritableReg)
+(rule (writable_regpair_lo (WritableRegPair.WritableRegPair _ lo)) lo)
+
+;; A (read-only) register pair.
+(type RegPair (enum (RegPair (hi Reg) (lo Reg))))
+
+;; Construct a register pair from a writable register pair.
+(decl writable_regpair_to_regpair (WritableRegPair) RegPair)
+(rule (writable_regpair_to_regpair (WritableRegPair.WritableRegPair hi lo))
+      (RegPair.RegPair (writable_reg_to_reg hi) (writable_reg_to_reg lo)))
+
+;; Uninitalized register pair that can be used for piecewise initialization.
+(decl uninitialized_regpair () RegPair)
+(rule (uninitialized_regpair)
+      (writable_regpair_to_regpair (temp_writable_regpair)))
+
+;; Retrieve the high word of the register pair.
+(decl regpair_hi (RegPair) Reg)
+(rule (regpair_hi (RegPair.RegPair hi _)) hi)
+
+;; Retrieve the low word of the register pair.
+(decl regpair_lo (RegPair) Reg)
+(rule (regpair_lo (RegPair.RegPair _ lo)) lo)
+
+
+;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Emit an instruction.
+;;
+;; This is low-level and side-effectful; it should only be used as an
+;; implementation detail by helpers that preserve the SSA facade themselves.
+(decl emit (MInst) Unit)
+(extern constructor emit emit)
+
+;; Helper for emitting `MInst.AluRRR` instructions.
+(decl alu_rrr (Type ALUOp Reg Reg) Reg)
+(rule (alu_rrr ty op src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.AluRRR op dst src1 src2))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRRSImm16` instructions.
+(decl alu_rrsimm16 (Type ALUOp Reg i16) Reg)
+(rule (alu_rrsimm16 ty op src imm)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.AluRRSImm16 op dst src imm))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRR` instructions.
+(decl alu_rr (Type ALUOp Reg Reg) Reg)
+(rule (alu_rr ty op src1 src2)
+      (let ((dst WritableReg (copy_writable_reg ty src1))
+            (_ Unit (emit (MInst.AluRR op dst src2))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRX` instructions.
+(decl alu_rx (Type ALUOp Reg MemArg) Reg)
+(rule (alu_rx ty op src mem)
+      (let ((dst WritableReg (copy_writable_reg ty src))
+            (_ Unit (emit (MInst.AluRX op dst mem))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRSImm16` instructions.
+(decl alu_rsimm16 (Type ALUOp Reg i16) Reg)
+(rule (alu_rsimm16 ty op src imm)
+      (let ((dst WritableReg (copy_writable_reg ty src))
+            (_ Unit (emit (MInst.AluRSImm16 op dst imm))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRSImm32` instructions.
+(decl alu_rsimm32 (Type ALUOp Reg i32) Reg)
+(rule (alu_rsimm32 ty op src imm)
+      (let ((dst WritableReg (copy_writable_reg ty src))
+            (_ Unit (emit (MInst.AluRSImm32 op dst imm))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRUImm32` instructions.
+(decl alu_ruimm32 (Type ALUOp Reg u32) Reg)
+(rule (alu_ruimm32 ty op src imm)
+      (let ((dst WritableReg (copy_writable_reg ty src))
+            (_ Unit (emit (MInst.AluRUImm32 op dst imm))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRUImm16Shifted` instructions.
+(decl alu_ruimm16shifted (Type ALUOp Reg UImm16Shifted) Reg)
+(rule (alu_ruimm16shifted ty op src imm)
+      (let ((dst WritableReg (copy_writable_reg ty src))
+            (_ Unit (emit (MInst.AluRUImm16Shifted op dst imm))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AluRUImm32Shifted` instructions.
+(decl alu_ruimm32shifted (Type ALUOp Reg UImm32Shifted) Reg)
+(rule (alu_ruimm32shifted ty op src imm)
+      (let ((dst WritableReg (copy_writable_reg ty src))
+            (_ Unit (emit (MInst.AluRUImm32Shifted op dst imm))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.SMulWide` instructions.
+(decl smul_wide (Reg Reg) RegPair)
+(rule (smul_wide src1 src2)
+      (let ((dst WritableRegPair (temp_writable_regpair))
+            (_ Unit (emit (MInst.SMulWide src1 src2))))
+        (writable_regpair_to_regpair dst)))
+
+;; Helper for emitting `MInst.UMulWide` instructions.
+(decl umul_wide (Reg Reg) RegPair)
+(rule (umul_wide src1 src2)
+      (let ((dst WritableRegPair (temp_writable_regpair))
+            (_1 Unit (emit (MInst.Mov64 (writable_regpair_lo dst) src2)))
+            (_2 Unit (emit (MInst.UMulWide src1))))
+        (writable_regpair_to_regpair dst)))
+
+;; Helper for emitting `MInst.SDivMod32` instructions.
+(decl sdivmod32 (RegPair Reg) RegPair)
+(rule (sdivmod32 src1 src2)
+      (let ((dst WritableRegPair (copy_writable_regpair src1))
+            (_ Unit (emit (MInst.SDivMod32 src2))))
+        (writable_regpair_to_regpair dst)))
+
+;; Helper for emitting `MInst.SDivMod64` instructions.
+(decl sdivmod64 (RegPair Reg) RegPair)
+(rule (sdivmod64 src1 src2)
+      (let ((dst WritableRegPair (copy_writable_regpair src1))
+            (_ Unit (emit (MInst.SDivMod64 src2))))
+        (writable_regpair_to_regpair dst)))
+
+;; Helper for emitting `MInst.UDivMod32` instructions.
+(decl udivmod32 (RegPair Reg) RegPair)
+(rule (udivmod32 src1 src2)
+      (let ((dst WritableRegPair (copy_writable_regpair src1))
+            (_ Unit (emit (MInst.UDivMod32 src2))))
+        (writable_regpair_to_regpair dst)))
+
+;; Helper for emitting `MInst.UDivMod64` instructions.
+(decl udivmod64 (RegPair Reg) RegPair)
+(rule (udivmod64 src1 src2)
+      (let ((dst WritableRegPair (copy_writable_regpair src1))
+            (_ Unit (emit (MInst.UDivMod64 src2))))
+        (writable_regpair_to_regpair dst)))
+
+;; Helper for emitting `MInst.ShiftRR` instructions.
+(decl shift_rr (Type ShiftOp Reg u8 Reg) Reg)
+(rule (shift_rr ty op src shift_imm shift_reg)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.ShiftRR op dst src shift_imm shift_reg))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.UnaryRR` instructions.
+(decl unary_rr (Type UnaryOp Reg) Reg)
+(rule (unary_rr ty op src)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.UnaryRR op dst src))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.CmpRR` instructions.
+(decl cmp_rr (CmpOp Reg Reg) ProducesFlags)
+(rule (cmp_rr op src1 src2)
+      (ProducesFlags.ProducesFlags (MInst.CmpRR op src1 src2)
+                                   (invalid_reg)))
+
+;; Helper for emitting `MInst.CmpRX` instructions.
+(decl cmp_rx (CmpOp Reg MemArg) ProducesFlags)
+(rule (cmp_rx op src mem)
+      (ProducesFlags.ProducesFlags (MInst.CmpRX op src mem)
+                                   (invalid_reg)))
+
+;; Helper for emitting `MInst.CmpRSImm16` instructions.
+(decl cmp_rsimm16 (CmpOp Reg i16) ProducesFlags)
+(rule (cmp_rsimm16 op src imm)
+      (ProducesFlags.ProducesFlags (MInst.CmpRSImm16 op src imm)
+                                   (invalid_reg)))
+
+;; Helper for emitting `MInst.CmpRSImm32` instructions.
+(decl cmp_rsimm32 (CmpOp Reg i32) ProducesFlags)
+(rule (cmp_rsimm32 op src imm)
+      (ProducesFlags.ProducesFlags (MInst.CmpRSImm32 op src imm)
+                                   (invalid_reg)))
+
+;; Helper for emitting `MInst.CmpRUImm32` instructions.
+(decl cmp_ruimm32 (CmpOp Reg u32) ProducesFlags)
+(rule (cmp_ruimm32 op src imm)
+      (ProducesFlags.ProducesFlags (MInst.CmpRUImm32 op src imm)
+                                   (invalid_reg)))
+
+;; Helper for emitting `MInst.AtomicRmw` instructions.
+(decl atomic_rmw_impl (Type ALUOp Reg MemArg) Reg)
+(rule (atomic_rmw_impl ty op src mem)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.AtomicRmw op dst src mem))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AtomicCas32` instructions.
+(decl atomic_cas32 (Reg Reg MemArg) Reg)
+(rule (atomic_cas32 src1 src2 mem)
+      (let ((dst WritableReg (copy_writable_reg $I32 src1))
+            (_ Unit (emit (MInst.AtomicCas32 dst src2 mem))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.AtomicCas64` instructions.
+(decl atomic_cas64 (Reg Reg MemArg) Reg)
+(rule (atomic_cas64 src1 src2 mem)
+      (let ((dst WritableReg (copy_writable_reg $I64 src1))
+            (_ Unit (emit (MInst.AtomicCas64 dst src2 mem))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.Fence` instructions.
+(decl fence_impl () SideEffectNoResult)
+(rule (fence_impl)
+      (SideEffectNoResult.Inst (MInst.Fence)))
+
+;; Helper for emitting `MInst.Load32` instructions.
+(decl load32 (MemArg) Reg)
+(rule (load32 addr)
+      (let ((dst WritableReg (temp_writable_reg $I32))
+            (_ Unit (emit (MInst.Load32 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.Load64` instructions.
+(decl load64 (MemArg) Reg)
+(rule (load64 addr)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.Load64 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.LoadRev16` instructions.
+(decl loadrev16 (MemArg) Reg)
+(rule (loadrev16 addr)
+      (let ((dst WritableReg (temp_writable_reg $I32))
+            (_ Unit (emit (MInst.LoadRev16 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.LoadRev32` instructions.
+(decl loadrev32 (MemArg) Reg)
+(rule (loadrev32 addr)
+      (let ((dst WritableReg (temp_writable_reg $I32))
+            (_ Unit (emit (MInst.LoadRev32 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.LoadRev64` instructions.
+(decl loadrev64 (MemArg) Reg)
+(rule (loadrev64 addr)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadRev64 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.Store8` instructions.
+(decl store8 (Reg MemArg) SideEffectNoResult)
+(rule (store8 src addr)
+      (SideEffectNoResult.Inst (MInst.Store8 src addr)))
+
+;; Helper for emitting `MInst.Store16` instructions.
+(decl store16 (Reg MemArg) SideEffectNoResult)
+(rule (store16 src addr)
+      (SideEffectNoResult.Inst (MInst.Store16 src addr)))
+
+;; Helper for emitting `MInst.Store32` instructions.
+(decl store32 (Reg MemArg) SideEffectNoResult)
+(rule (store32 src addr)
+      (SideEffectNoResult.Inst (MInst.Store32 src addr)))
+
+;; Helper for emitting `MInst.Store64` instructions.
+(decl store64 (Reg MemArg) SideEffectNoResult)
+(rule (store64 src addr)
+      (SideEffectNoResult.Inst (MInst.Store64 src addr)))
+
+;; Helper for emitting `MInst.StoreImm8` instructions.
+(decl store8_imm (u8 MemArg) SideEffectNoResult)
+(rule (store8_imm imm addr)
+      (SideEffectNoResult.Inst (MInst.StoreImm8 imm addr)))
+
+;; Helper for emitting `MInst.StoreImm16` instructions.
+(decl store16_imm (i16 MemArg) SideEffectNoResult)
+(rule (store16_imm imm addr)
+      (SideEffectNoResult.Inst (MInst.StoreImm16 imm addr)))
+
+;; Helper for emitting `MInst.StoreImm32SExt16` instructions.
+(decl store32_simm16 (i16 MemArg) SideEffectNoResult)
+(rule (store32_simm16 imm addr)
+      (SideEffectNoResult.Inst (MInst.StoreImm32SExt16 imm addr)))
+
+;; Helper for emitting `MInst.StoreImm64SExt16` instructions.
+(decl store64_simm16 (i16 MemArg) SideEffectNoResult)
+(rule (store64_simm16 imm addr)
+      (SideEffectNoResult.Inst (MInst.StoreImm64SExt16 imm addr)))
+
+;; Helper for emitting `MInst.StoreRev16` instructions.
+(decl storerev16 (Reg MemArg) SideEffectNoResult)
+(rule (storerev16 src addr)
+      (SideEffectNoResult.Inst (MInst.StoreRev16 src addr)))
+
+;; Helper for emitting `MInst.StoreRev32` instructions.
+(decl storerev32 (Reg MemArg) SideEffectNoResult)
+(rule (storerev32 src addr)
+      (SideEffectNoResult.Inst (MInst.StoreRev32 src addr)))
+
+;; Helper for emitting `MInst.StoreRev64` instructions.
+(decl storerev64 (Reg MemArg) SideEffectNoResult)
+(rule (storerev64 src addr)
+      (SideEffectNoResult.Inst (MInst.StoreRev64 src addr)))
+
+;; Helper for emitting `MInst.FpuRR` instructions.
+(decl fpu_rr (Type FPUOp1 Reg) Reg)
+(rule (fpu_rr ty op src)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.FpuRR op dst src))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuRRR` instructions.
+(decl fpu_rrr (Type FPUOp2 Reg Reg) Reg)
+(rule (fpu_rrr ty op src1 src2)
+      (let ((dst WritableReg (copy_writable_reg ty src1))
+            (_ Unit (emit (MInst.FpuRRR op dst src2))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuRRRR` instructions.
+(decl fpu_rrrr (Type FPUOp3 Reg Reg Reg) Reg)
+(rule (fpu_rrrr ty op src1 src2 src3)
+      (let ((dst WritableReg (copy_writable_reg ty src1))
+            (_ Unit (emit (MInst.FpuRRRR op dst src2 src3))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuCopysign` instructions.
+(decl fpu_copysign (Type Reg Reg) Reg)
+(rule (fpu_copysign ty src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.FpuCopysign dst src1 src2))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuCmp32` instructions.
+(decl fpu_cmp32 (Reg Reg) ProducesFlags)
+(rule (fpu_cmp32 src1 src2)
+      (ProducesFlags.ProducesFlags (MInst.FpuCmp32 src1 src2)
+                                   (invalid_reg)))
+
+;; Helper for emitting `MInst.FpuCmp64` instructions.
+(decl fpu_cmp64 (Reg Reg) ProducesFlags)
+(rule (fpu_cmp64 src1 src2)
+      (ProducesFlags.ProducesFlags (MInst.FpuCmp64 src1 src2)
+                                   (invalid_reg)))
+
+;; Helper for emitting `MInst.FpuToInt` instructions.
+(decl fpu_to_int (Type FpuToIntOp Reg) ProducesFlags)
+(rule (fpu_to_int ty op src)
+      (let ((dst WritableReg (temp_writable_reg ty)))
+        (ProducesFlags.ProducesFlags (MInst.FpuToInt op dst src)
+                                     (writable_reg_to_reg dst))))
+
+;; Helper for emitting `MInst.IntToFpu` instructions.
+(decl int_to_fpu (Type IntToFpuOp Reg) Reg)
+(rule (int_to_fpu ty op src)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.IntToFpu op dst src))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuRound` instructions.
+(decl fpu_round (Type FpuRoundMode Reg) Reg)
+(rule (fpu_round ty mode src)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.FpuRound mode dst src))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuVecRRR` instructions.
+(decl fpuvec_rrr (Type FPUOp2 Reg Reg) Reg)
+(rule (fpuvec_rrr ty op src1 src2)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit (MInst.FpuVecRRR op dst src1 src2))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.MovToFpr` instructions.
+(decl mov_to_fpr (Reg) Reg)
+(rule (mov_to_fpr src)
+      (let ((dst WritableReg (temp_writable_reg $F64))
+            (_ Unit (emit (MInst.MovToFpr dst src))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.MovFromFpr` instructions.
+(decl mov_from_fpr (Reg) Reg)
+(rule (mov_from_fpr src)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.MovFromFpr dst src))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuLoad32` instructions.
+(decl fpu_load32 (MemArg) Reg)
+(rule (fpu_load32 addr)
+      (let ((dst WritableReg (temp_writable_reg $F32))
+            (_ Unit (emit (MInst.FpuLoad32 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuLoad64` instructions.
+(decl fpu_load64 (MemArg) Reg)
+(rule (fpu_load64 addr)
+      (let ((dst WritableReg (temp_writable_reg $F64))
+            (_ Unit (emit (MInst.FpuLoad64 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuLoadRev32` instructions.
+(decl fpu_loadrev32 (MemArg) Reg)
+(rule (fpu_loadrev32 addr)
+      (let ((dst WritableReg (temp_writable_reg $F32))
+            (_ Unit (emit (MInst.FpuLoadRev32 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuLoadRev64` instructions.
+(decl fpu_loadrev64 (MemArg) Reg)
+(rule (fpu_loadrev64 addr)
+      (let ((dst WritableReg (temp_writable_reg $F64))
+            (_ Unit (emit (MInst.FpuLoadRev64 dst addr))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.FpuStore32` instructions.
+(decl fpu_store32 (Reg MemArg) SideEffectNoResult)
+(rule (fpu_store32 src addr)
+      (SideEffectNoResult.Inst (MInst.FpuStore32 src addr)))
+
+;; Helper for emitting `MInst.FpuStore64` instructions.
+(decl fpu_store64 (Reg MemArg) SideEffectNoResult)
+(rule (fpu_store64 src addr)
+      (SideEffectNoResult.Inst (MInst.FpuStore64 src addr)))
+
+;; Helper for emitting `MInst.FpuStoreRev32` instructions.
+(decl fpu_storerev32 (Reg MemArg) SideEffectNoResult)
+(rule (fpu_storerev32 src addr)
+      (SideEffectNoResult.Inst (MInst.FpuStoreRev32 src addr)))
+
+;; Helper for emitting `MInst.FpuStoreRev64` instructions.
+(decl fpu_storerev64 (Reg MemArg) SideEffectNoResult)
+(rule (fpu_storerev64 src addr)
+      (SideEffectNoResult.Inst (MInst.FpuStoreRev64 src addr)))
+
+;; Helper for emitting `MInst.LoadExtNameFar` instructions.
+(decl load_ext_name_far (BoxExternalName i64) Reg)
+(rule (load_ext_name_far name offset)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadExtNameFar dst name offset))))
+        (writable_reg_to_reg dst)))
+
+;; Helper for emitting `MInst.LoadAddr` instructions.
+(decl load_addr (MemArg) Reg)
+(rule (load_addr mem)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadAddr dst mem))))
+        (writable_reg_to_reg dst)))
+
+;; Emit a `ProducesFlags` instruction when the flags are not actually needed.
+(decl drop_flags (ProducesFlags) Reg)
+(rule (drop_flags (ProducesFlags.ProducesFlags inst result))
+      (let ((_ Unit (emit inst)))
+        result))
+
+
+;; Helpers for generating register moves ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Move source register into destination.  (Non-SSA form.)
+(decl emit_mov (Type WritableReg Reg) Unit)
+
+(rule (emit_mov (gpr32_ty _ty) dst src)
+      (emit (MInst.Mov32 dst src)))
+
+(rule (emit_mov (gpr64_ty _ty) dst src)
+      (emit (MInst.Mov64 dst src)))
+
+(rule (emit_mov $F32 dst src)
+      (emit (MInst.FpuMove32 dst src)))
+
+(rule (emit_mov $F64 dst src)
+      (emit (MInst.FpuMove64 dst src)))
+
+;; Allocate a temporary (writable) register, initialized as a copy of the input.
+(decl copy_writable_reg (Type Reg) WritableReg)
+(rule (copy_writable_reg ty src)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit_mov ty dst src)))
+        dst))
+
+;; Likewise, but returning a Reg instead of a WritableReg.
+(decl copy_reg (Type Reg) Reg)
+(rule (copy_reg ty reg) (writable_reg_to_reg (copy_writable_reg ty reg)))
+
+
+;; Helpers for generating immediate values ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Move immediate value into destination register.  (Non-SSA form.)
+(decl emit_imm (Type WritableReg u64) Unit)
+
+;; 16-bit (or smaller) result type, any value
+(rule (emit_imm (fits_in_16 _ty) dst n)
+      (emit (MInst.Mov32SImm16 dst (u64_as_i16 n))))
+
+;; 32-bit result type, value fits in i16
+(rule (emit_imm (gpr32_ty _ty) dst (i16_from_u64 n))
+      (emit (MInst.Mov32SImm16 dst n)))
+
+;; 32-bit result type, any value
+(rule (emit_imm (gpr32_ty _ty) dst n)
+      (emit (MInst.Mov32Imm dst (u64_as_u32 n))))
+
+;; 64-bit result type, value fits in i16
+(rule (emit_imm (gpr64_ty _ty) dst (i16_from_u64 n))
+      (emit (MInst.Mov64SImm16 dst n)))
+
+;; 64-bit result type, value fits in i32
+(rule (emit_imm (gpr64_ty _ty) dst (i32_from_u64 n))
+      (emit (MInst.Mov64SImm32 dst n)))
+
+;; 64-bit result type, value fits in UImm16Shifted
+(rule (emit_imm (gpr64_ty _ty) dst (uimm16shifted_from_u64 n))
+      (emit (MInst.Mov64UImm16Shifted dst n)))
+
+;; 64-bit result type, value fits in UImm32Shifted
+(rule (emit_imm (gpr64_ty _ty) dst (uimm32shifted_from_u64 n))
+      (emit (MInst.Mov64UImm32Shifted dst n)))
+
+;; 64-bit result type, value with non-zero low-/high-parts.
+(rule (emit_imm (gpr64_ty ty) dst (and (u64_nonzero_hipart hi)
+                                       (u64_nonzero_lopart lo)))
+      (let ((_ Unit (emit_imm ty dst hi)))
+        (emit_insert_imm dst lo)))
+
+;; Insert immediate value into destination register.  (Non-SSA form.)
+(decl emit_insert_imm (WritableReg u64) Unit)
+
+;; Insertion, value fits in UImm16Shifted
+(rule (emit_insert_imm dst (uimm16shifted_from_u64 n))
+      (emit (MInst.Insert64UImm16Shifted dst n)))
+
+;; Insertion, value fits in UImm32Shifted
+(rule (emit_insert_imm dst (uimm32shifted_from_u64 n))
+      (emit (MInst.Insert64UImm32Shifted dst n)))
+
+;; 32-bit floating-point type, any value.  Loaded from literal pool.
+;; TODO: use LZER to load 0.0
+(rule (emit_imm $F32 dst n)
+      (emit (MInst.LoadFpuConst32 dst (u64_as_u32 n))))
+
+;; 64-bit floating-point type, any value.  Loaded from literal pool.
+;; TODO: use LZDR to load 0.0
+(rule (emit_imm $F64 dst n)
+      (emit (MInst.LoadFpuConst64 dst n)))
+
+;; Allocate a temporary register, initialized with an immediate.
+(decl imm (Type u64) Reg)
+(rule (imm ty n)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_ Unit (emit_imm ty dst n)))
+        (writable_reg_to_reg dst)))
+
+;; Place an immediate into the low half of a register pair.
+;; The high half is taken from the input.
+(decl imm_regpair_lo (Type u64 RegPair) RegPair)
+(rule (imm_regpair_lo ty n regpair)
+      (let ((dst WritableRegPair (copy_writable_regpair regpair))
+            (_ Unit (emit_imm ty (writable_regpair_lo dst) n)))
+        (writable_regpair_to_regpair dst)))
+
+;; Place an immediate into the high half of a register pair.
+;; The low half is taken from the input.
+(decl imm_regpair_hi (Type u64 RegPair) RegPair)
+(rule (imm_regpair_hi ty n regpair)
+      (let ((dst WritableRegPair (copy_writable_regpair regpair))
+            (_ Unit (emit_imm ty (writable_regpair_hi dst) n)))
+        (writable_regpair_to_regpair dst)))
+
+
+;; Helpers for generating extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Result type of extending integer `Type` to 32 bits if smaller.
+(decl ty_ext32 (Type) Type)
+(rule (ty_ext32 $I8) $I32)
+(rule (ty_ext32 $I16) $I32)
+(rule (ty_ext32 $I32) $I32)
+(rule (ty_ext32 $I64) $I64)
+
+;; Result type of extending integer `Type` to 64 bits if smaller.
+(decl ty_ext64 (Type) Type)
+(rule (ty_ext64 $I8) $I64)
+(rule (ty_ext64 $I16) $I64)
+(rule (ty_ext64 $I32) $I64)
+(rule (ty_ext64 $I64) $I64)
+
+;; Zero-extend a register from a smaller `Type` into a 32-bit destination.  (Non-SSA form.)
+;; This handles both integer and boolean input types.
+(decl emit_zext32_reg (WritableReg Type Reg) Unit)
+(rule (emit_zext32_reg dst ty src)
+      (emit (MInst.Extend dst src $false (ty_bits ty) 32)))
+
+;; Sign-extend a register from a smaller `Type` into a 32-bit destination.  (Non-SSA form.)
+;; This handles both integer and boolean input types.
+(decl emit_sext32_reg (WritableReg Type Reg) Unit)
+(rule (emit_sext32_reg dst ty src)
+      (emit (MInst.Extend dst src $true (ty_bits ty) 32)))
+
+;; Zero-extend a register from a smaller `Type` into a 64-bit destination.  (Non-SSA form.)
+;; This handles both integer and boolean input types.
+(decl emit_zext64_reg (WritableReg Type Reg) Unit)
+(rule (emit_zext64_reg dst ty src)
+      (emit (MInst.Extend dst src $false (ty_bits ty) 64)))
+
+;; Sign-extend a register from a smaller `Type` into a 64-bit destination.  (Non-SSA form.)
+;; This handles both integer and boolean input types.
+(decl emit_sext64_reg (WritableReg Type Reg) Unit)
+(rule (emit_sext64_reg dst ty src)
+      (emit (MInst.Extend dst src $true (ty_bits ty) 64)))
+
+;; Zero-extend a register from a smaller `Type` into a 32-bit register.
+;; This handles both integer and boolean input types.
+(decl zext32_reg (Type Reg) Reg)
+(rule (zext32_reg ty src)
+      (let ((dst WritableReg (temp_writable_reg $I32))
+            (_ Unit (emit_zext32_reg dst ty src)))
+        (writable_reg_to_reg dst)))
+
+;; Sign-extend a register from a smaller `Type` into a 32-bit register.
+;; This handles both integer and boolean input types (except $B1).
+(decl sext32_reg (Type Reg) Reg)
+(rule (sext32_reg ty src)
+      (let ((dst WritableReg (temp_writable_reg $I32))
+            (_ Unit (emit_sext32_reg dst ty src)))
+        (writable_reg_to_reg dst)))
+
+;; Zero-extend a register from a smaller `Type` into a 64-bit register.
+;; This handles both integer and boolean input types (except $B1).
+(decl zext64_reg (Type Reg) Reg)
+(rule (zext64_reg ty src)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit_zext64_reg dst ty src)))
+        (writable_reg_to_reg dst)))
+
+;; Sign-extend a register from a smaller `Type` into a 64-bit register.
+;; This handles both integer and boolean input types (except $B1).
+(decl sext64_reg (Type Reg) Reg)
+(rule (sext64_reg ty src)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit_sext64_reg dst ty src)))
+        (writable_reg_to_reg dst)))
+
+
+;; Zero-extend memory from a smaller `Type` into a 32-bit destination.  (Non-SSA form.)
+(decl emit_zext32_mem (WritableReg Type MemArg) Unit)
+(rule (emit_zext32_mem dst $I8 mem) (emit (MInst.Load32ZExt8 dst mem)))
+(rule (emit_zext32_mem dst $I16 mem) (emit (MInst.Load32ZExt16 dst mem)))
+
+;; Sign-extend memory from a smaller `Type` into a 32-bit destination.  (Non-SSA form.)
+(decl emit_sext32_mem (WritableReg Type MemArg) Unit)
+(rule (emit_sext32_mem dst $I8 mem) (emit (MInst.Load32SExt8 dst mem)))
+(rule (emit_sext32_mem dst $I16 mem) (emit (MInst.Load32SExt16 dst mem)))
+
+;; Zero-extend memory from a smaller `Type` into a 64-bit destination.  (Non-SSA form.)
+(decl emit_zext64_mem (WritableReg Type MemArg) Unit)
+(rule (emit_zext64_mem dst $I8 mem) (emit (MInst.Load64ZExt8 dst mem)))
+(rule (emit_zext64_mem dst $I16 mem) (emit (MInst.Load64ZExt16 dst mem)))
+(rule (emit_zext64_mem dst $I32 mem) (emit (MInst.Load64ZExt32 dst mem)))
+
+;; Sign-extend memory from a smaller `Type` into a 64-bit destination.  (Non-SSA form.)
+(decl emit_sext64_mem (WritableReg Type MemArg) Unit)
+(rule (emit_sext64_mem dst $I8 mem) (emit (MInst.Load64SExt8 dst mem)))
+(rule (emit_sext64_mem dst $I16 mem) (emit (MInst.Load64SExt16 dst mem)))
+(rule (emit_sext64_mem dst $I32 mem) (emit (MInst.Load64SExt32 dst mem)))
+
+;; Zero-extend memory from a smaller `Type` into a 32-bit register.
+(decl zext32_mem (Type MemArg) Reg)
+(rule (zext32_mem ty mem)
+      (let ((dst WritableReg (temp_writable_reg $I32))
+            (_ Unit (emit_zext32_mem dst ty mem)))
+        (writable_reg_to_reg dst)))
+
+;; Sign-extend memory from a smaller `Type` into a 32-bit register.
+(decl sext32_mem (Type MemArg) Reg)
+(rule (sext32_mem ty mem)
+      (let ((dst WritableReg (temp_writable_reg $I32))
+            (_ Unit (emit_sext32_mem dst ty mem)))
+        (writable_reg_to_reg dst)))
+
+;; Zero-extend memory from a smaller `Type` into a 64-bit register.
+(decl zext64_mem (Type MemArg) Reg)
+(rule (zext64_mem ty mem)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit_zext64_mem dst ty mem)))
+        (writable_reg_to_reg dst)))
+
+;; Sign-extend memory from a smaller `Type` into a 64-bit register.
+(decl sext64_mem (Type MemArg) Reg)
+(rule (sext64_mem ty mem)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit_sext64_mem dst ty mem)))
+        (writable_reg_to_reg dst)))
+
+
+;; Place `Value` into destination, zero-extending to 32 bits if smaller.  (Non-SSA form.)
+(decl emit_put_in_reg_zext32 (WritableReg Value) Unit)
+(rule (emit_put_in_reg_zext32 dst (and (value_type ty) (u64_from_value val)))
+      (emit_imm (ty_ext32 ty) dst val))
+(rule (emit_put_in_reg_zext32 dst (and (value_type (fits_in_16 ty)) (sinkable_load load)))
+      (emit_zext32_mem dst ty (sink_load load)))
+(rule (emit_put_in_reg_zext32 dst val @ (value_type (fits_in_16 ty)))
+      (emit_zext32_reg dst ty (put_in_reg val)))
+(rule (emit_put_in_reg_zext32 dst val @ (value_type (ty_32_or_64 ty)))
+      (emit_mov ty dst (put_in_reg val)))
+
+;; Place `Value` into destination, sign-extending to 32 bits if smaller.  (Non-SSA form.)
+(decl emit_put_in_reg_sext32 (WritableReg Value) Unit)
+(rule (emit_put_in_reg_sext32 dst (and (value_type ty) (u64_from_signed_value val)))
+      (emit_imm (ty_ext32 ty) dst val))
+(rule (emit_put_in_reg_sext32 dst (and (value_type (fits_in_16 ty)) (sinkable_load load)))
+      (emit_sext32_mem dst ty (sink_load load)))
+(rule (emit_put_in_reg_sext32 dst val @ (value_type (fits_in_16 ty)))
+      (emit_sext32_reg dst ty (put_in_reg val)))
+(rule (emit_put_in_reg_sext32 dst val @ (value_type (ty_32_or_64 ty)))
+      (emit_mov ty dst (put_in_reg val)))
+
+;; Place `Value` into destination, zero-extending to 64 bits if smaller.  (Non-SSA form.)
+(decl emit_put_in_reg_zext64 (WritableReg Value) Unit)
+(rule (emit_put_in_reg_zext64 dst (and (value_type ty) (u64_from_value val)))
+      (emit_imm (ty_ext64 ty) dst val))
+(rule (emit_put_in_reg_zext64 dst (and (value_type (gpr32_ty ty)) (sinkable_load load)))
+      (emit_zext64_mem dst ty (sink_load load)))
+(rule (emit_put_in_reg_zext64 dst val @ (value_type (gpr32_ty ty)))
+      (emit_zext64_reg dst ty (put_in_reg val)))
+(rule (emit_put_in_reg_zext64 dst val @ (value_type (gpr64_ty ty)))
+      (emit_mov ty dst (put_in_reg val)))
+
+;; Place `Value` into destination, sign-extending to 64 bits if smaller.  (Non-SSA form.)
+(decl emit_put_in_reg_sext64 (WritableReg Value) Unit)
+(rule (emit_put_in_reg_sext64 dst (and (value_type ty) (u64_from_signed_value val)))
+      (emit_imm (ty_ext64 ty) dst val))
+(rule (emit_put_in_reg_sext64 dst (and (value_type (gpr32_ty ty)) (sinkable_load load)))
+      (emit_sext64_mem dst ty (sink_load load)))
+(rule (emit_put_in_reg_sext64 dst val @ (value_type (gpr32_ty ty)))
+      (emit_sext64_reg dst ty (put_in_reg val)))
+(rule (emit_put_in_reg_sext64 dst val @ (value_type (gpr64_ty ty)))
+      (emit_mov ty dst (put_in_reg val)))
+
+;; Place `Value` into a register, zero-extending to 32 bits if smaller.
+(decl put_in_reg_zext32 (Value) Reg)
+(rule (put_in_reg_zext32 (and (value_type ty) (u64_from_value val)))
+      (imm (ty_ext32 ty) val))
+(rule (put_in_reg_zext32 (and (value_type (fits_in_16 ty)) (sinkable_load load)))
+      (zext32_mem ty (sink_load load)))
+(rule (put_in_reg_zext32 val @ (value_type (fits_in_16 ty)))
+      (zext32_reg ty (put_in_reg val)))
+(rule (put_in_reg_zext32 val @ (value_type (ty_32_or_64 _ty)))
+      (put_in_reg val))
+
+;; Place `Value` into a register, sign-extending to 32 bits if smaller.
+(decl put_in_reg_sext32 (Value) Reg)
+(rule (put_in_reg_sext32 (and (value_type ty) (u64_from_signed_value val)))
+      (imm (ty_ext32 ty) val))
+(rule (put_in_reg_sext32 (and (value_type (fits_in_16 ty)) (sinkable_load load)))
+      (sext32_mem ty (sink_load load)))
+(rule (put_in_reg_sext32 val @ (value_type (fits_in_16 ty)))
+      (sext32_reg ty (put_in_reg val)))
+(rule (put_in_reg_sext32 val @ (value_type (ty_32_or_64 _ty)))
+      (put_in_reg val))
+
+;; Place `Value` into a register, zero-extending to 64 bits if smaller.
+(decl put_in_reg_zext64 (Value) Reg)
+(rule (put_in_reg_zext64 (and (value_type ty) (u64_from_value val)))
+      (imm (ty_ext64 ty) val))
+(rule (put_in_reg_zext64 (and (value_type (gpr32_ty ty)) (sinkable_load load)))
+      (zext64_mem ty (sink_load load)))
+(rule (put_in_reg_zext64 val @ (value_type (gpr32_ty ty)))
+      (zext64_reg ty (put_in_reg val)))
+(rule (put_in_reg_zext64 val @ (value_type (gpr64_ty ty)))
+      (put_in_reg val))
+
+;; Place `Value` into a register, sign-extending to 64 bits if smaller.
+(decl put_in_reg_sext64 (Value) Reg)
+(rule (put_in_reg_sext64 (and (value_type ty) (u64_from_signed_value val)))
+      (imm (ty_ext64 ty) val))
+(rule (put_in_reg_sext64 (and (value_type (gpr32_ty ty)) (sinkable_load load)))
+      (sext64_mem ty (sink_load load)))
+(rule (put_in_reg_sext64 val @ (value_type (gpr32_ty ty)))
+      (sext64_reg ty (put_in_reg val)))
+(rule (put_in_reg_sext64 val @ (value_type (gpr64_ty ty)))
+      (put_in_reg val))
+
+;; Place `Value` into the low half of a register pair, zero-extending
+;; to 32 bits if smaller.  The high half is taken from the input.
+(decl put_in_regpair_lo_zext32 (Value RegPair) RegPair)
+(rule (put_in_regpair_lo_zext32 val regpair)
+      (let ((dst WritableRegPair (copy_writable_regpair regpair))
+            (_ Unit (emit_put_in_reg_zext32 (writable_regpair_lo dst) val)))
+        (writable_regpair_to_regpair dst)))
+
+;; Place `Value` into the low half of a register pair, sign-extending
+;; to 32 bits if smaller.  The high half is taken from the input.
+(decl put_in_regpair_lo_sext32 (Value RegPair) RegPair)
+(rule (put_in_regpair_lo_sext32 val regpair)
+      (let ((dst WritableRegPair (copy_writable_regpair regpair))
+            (_ Unit (emit_put_in_reg_sext32 (writable_regpair_lo dst) val)))
+        (writable_regpair_to_regpair dst)))
+
+;; Place `Value` into the low half of a register pair, zero-extending
+;; to 64 bits if smaller.  The high half is taken from the input.
+(decl put_in_regpair_lo_zext64 (Value RegPair) RegPair)
+(rule (put_in_regpair_lo_zext64 val regpair)
+      (let ((dst WritableRegPair (copy_writable_regpair regpair))
+            (_ Unit (emit_put_in_reg_zext64 (writable_regpair_lo dst) val)))
+        (writable_regpair_to_regpair dst)))
+
+;; Place `Value` into the low half of a register pair, sign-extending
+;; to 64 bits if smaller.  The high half is taken from the input.
+(decl put_in_regpair_lo_sext64 (Value RegPair) RegPair)
+(rule (put_in_regpair_lo_sext64 val regpair)
+      (let ((dst WritableRegPair (copy_writable_regpair regpair))
+            (_ Unit (emit_put_in_reg_sext64 (writable_regpair_lo dst) val)))
+        (writable_regpair_to_regpair dst)))
+
+
+;; Helpers for generating conditional moves ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Conditionally move immediate value into destination register.  (Non-SSA form.)
+(decl emit_cmov_imm (Type WritableReg Cond i16) ConsumesFlags)
+(rule (emit_cmov_imm (gpr32_ty _ty) dst cond imm)
+      (ConsumesFlags.ConsumesFlags (MInst.CMov32SImm16 dst cond imm)
+                                   (writable_reg_to_reg dst)))
+(rule (emit_cmov_imm (gpr64_ty _ty) dst cond imm)
+      (ConsumesFlags.ConsumesFlags (MInst.CMov64SImm16 dst cond imm)
+                                   (writable_reg_to_reg dst)))
+
+;; Conditionally select between immediate and source register.
+(decl cmov_imm (Type Cond i16 Reg) ConsumesFlags)
+(rule (cmov_imm ty cond imm src)
+      (let ((dst WritableReg (copy_writable_reg ty src)))
+        (emit_cmov_imm ty dst cond imm)))
+
+;; Conditionally modify the low word of a register pair.
+;; This cannot be ConsumesFlags since the return value is not a register.
+(decl cmov_imm_regpair_lo (Type ProducesFlags Cond i16 RegPair) RegPair)
+(rule (cmov_imm_regpair_lo ty producer cond imm src)
+      (let ((dst WritableRegPair (copy_writable_regpair src))
+            (consumer ConsumesFlags (emit_cmov_imm ty (writable_regpair_lo dst) cond imm))
+            (_ Reg (with_flags_1 producer consumer)))
+        (writable_regpair_to_regpair dst)))
+
+;; Conditionally modify the high word of a register pair.
+;; This cannot be ConsumesFlags since the return value is not a register.
+(decl cmov_imm_regpair_hi (Type ProducesFlags Cond i16 RegPair) RegPair)
+(rule (cmov_imm_regpair_hi ty producer cond imm src)
+      (let ((dst WritableRegPair (copy_writable_regpair src))
+            (consumer ConsumesFlags (emit_cmov_imm ty (writable_regpair_hi dst) cond imm))
+            (_ Reg (with_flags_1 producer consumer)))
+        (writable_regpair_to_regpair dst)))
+
+;; Conditionally select between two source registers.  (Non-SSA form.)
+(decl emit_cmov_reg (Type WritableReg Cond Reg) ConsumesFlags)
+(rule (emit_cmov_reg (gpr32_ty _ty) dst cond src)
+      (ConsumesFlags.ConsumesFlags (MInst.CMov32 dst cond src)
+                                   (writable_reg_to_reg dst)))
+(rule (emit_cmov_reg (gpr64_ty _ty) dst cond src)
+      (ConsumesFlags.ConsumesFlags (MInst.CMov64 dst cond src)
+                                   (writable_reg_to_reg dst)))
+(rule (emit_cmov_reg $F32 dst cond src)
+      (ConsumesFlags.ConsumesFlags (MInst.FpuCMov32 dst cond src)
+                                   (writable_reg_to_reg dst)))
+(rule (emit_cmov_reg $F64 dst cond src)
+      (ConsumesFlags.ConsumesFlags (MInst.FpuCMov64 dst cond src)
+                                   (writable_reg_to_reg dst)))
+
+;; Conditionally select between two source registers.
+(decl cmov_reg (Type Cond Reg Reg) ConsumesFlags)
+(rule (cmov_reg ty cond src1 src2)
+      (let ((dst WritableReg (copy_writable_reg ty src2)))
+        (emit_cmov_reg ty dst cond src1)))
+
+
+;; Helpers for generating conditional traps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl trap_if (ProducesFlags Cond TrapCode) Reg)
+(rule (trap_if (ProducesFlags.ProducesFlags inst result) cond trap_code)
+      (let ((_1 Unit (emit inst))
+            (_2 Unit (emit (MInst.TrapIf cond trap_code))))
+        result))
+
+(decl icmps_reg_and_trap (Type Reg Reg Cond TrapCode) Reg)
+(rule (icmps_reg_and_trap ty src1 src2 cond trap_code)
+      (let ((_ Unit (emit (MInst.CmpTrapRR (cmpop_cmps ty)
+                                           src1 src2 cond trap_code))))
+        (invalid_reg)))
+
+(decl icmps_simm16_and_trap (Type Reg i16 Cond TrapCode) Reg)
+(rule (icmps_simm16_and_trap ty src imm cond trap_code)
+      (let ((_ Unit (emit (MInst.CmpTrapRSImm16 (cmpop_cmps ty)
+                                                src imm cond trap_code))))
+        (invalid_reg)))
+
+(decl icmpu_reg_and_trap (Type Reg Reg Cond TrapCode) Reg)
+(rule (icmpu_reg_and_trap ty src1 src2 cond trap_code)
+      (let ((_ Unit (emit (MInst.CmpTrapRR (cmpop_cmpu ty)
+                                           src1 src2 cond trap_code))))
+        (invalid_reg)))
+
+(decl icmpu_uimm16_and_trap (Type Reg u16 Cond TrapCode) Reg)
+(rule (icmpu_uimm16_and_trap ty src imm cond trap_code)
+      (let ((_ Unit (emit (MInst.CmpTrapRUImm16 (cmpop_cmpu ty)
+                                                src imm cond trap_code))))
+        (invalid_reg)))
+
+
+;; Helpers for handling boolean conditions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Capture a flags-producing instruction together with a condition code mask
+;; that tells how to interpret the resulting condition code.  The result can
+;; be used to represent a boolean condition.
+(type ProducesBool (enum (ProducesBool (producer ProducesFlags) (cond Cond))))
+(decl bool (ProducesFlags Cond) ProducesBool)
+(rule (bool producer cond) (ProducesBool.ProducesBool producer cond))
+
+;; Invert boolean condition.
+(decl invert_bool (ProducesBool) ProducesBool)
+(rule (invert_bool (ProducesBool.ProducesBool producer cond))
+      (bool producer (invert_cond cond)))
+
+;; Helpers to emit a `ProducesFlags` or `ConsumesFlags` instruction directly.
+;; We use this in `select_bool_reg` and `select_bool_imm` below instead of
+;; using the `with_flags` mechanism so that we can insert another unrelated
+;; instruction in between the producer and consumer.  (This use is only valid
+;; if that unrelated instruction does not modify the condition code.)
+(decl emit_producer (ProducesFlags) Unit)
+(rule (emit_producer (ProducesFlags.ProducesFlags insn _)) (emit insn))
+(decl emit_consumer (ConsumesFlags) Unit)
+(rule (emit_consumer (ConsumesFlags.ConsumesFlags insn _)) (emit insn))
+
+;; Use a boolean condition to select between two registers.
+(decl select_bool_reg (Type ProducesBool Reg Reg) Reg)
+(rule (select_bool_reg ty (ProducesBool.ProducesBool producer cond) reg_true reg_false)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_1 Unit (emit_producer producer))
+            (_2 Unit (emit_mov ty dst reg_false))
+            (_3 Unit (emit_consumer (emit_cmov_reg ty dst cond reg_true))))
+        (writable_reg_to_reg dst)))
+
+;; Use a boolean condition to select between two immediate values.
+(decl select_bool_imm (Type ProducesBool i16 u64) Reg)
+(rule (select_bool_imm ty (ProducesBool.ProducesBool producer cond) imm_true imm_false)
+      (let ((dst WritableReg (temp_writable_reg ty))
+            (_1 Unit (emit_producer producer))
+            (_2 Unit (emit_imm ty dst imm_false))
+            (_3 Unit (emit_consumer (emit_cmov_imm ty dst cond imm_true))))
+        (writable_reg_to_reg dst)))
+
+;; Lower a boolean condition to a boolean type.  The value used to represent
+;; "true" is -1 for all result types except for $B1, which uses 1.
+(decl lower_bool (Type ProducesBool) Reg)
+(rule (lower_bool $B1 cond) (select_bool_imm $B1 cond 1 0))
+(rule (lower_bool $B8 cond) (select_bool_imm $B8 cond -1 0))
+(rule (lower_bool $B16 cond) (select_bool_imm $B16 cond -1 0))
+(rule (lower_bool $B32 cond) (select_bool_imm $B32 cond -1 0))
+(rule (lower_bool $B64 cond) (select_bool_imm $B64 cond -1 0))
+
+
+;; Helpers for generating `clz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Count leading zeroes.  For a zero input, return the specified value.
+(decl clz_reg (i16 Reg) RegPair)
+
+;; The flogr instruction returns 64 for zero input by default.
+(rule (clz_reg 64 x)
+      (let ((dst WritableRegPair (temp_writable_regpair))
+            (_ Unit (emit (MInst.Flogr x))))
+        (writable_regpair_to_regpair dst)))
+
+;; If another zero return value was requested, we need to override the flogr
+;; result.  This cannot use any of the normal flags mechanisms because we need
+;; to use both result and condition code output of flogr as input to the
+;; conditional move, and because flogr returns a register pair.
+(rule (clz_reg zeroval x)
+      (let ((dst WritableRegPair (temp_writable_regpair))
+            (_1 Unit (emit (MInst.Flogr x)))
+            (_2 Unit (emit (MInst.CMov64SImm16 (writable_regpair_hi dst)
+                              (intcc_as_cond (IntCC.Equal)) zeroval))))
+        (writable_regpair_to_regpair dst)))
+
+
+;; Helpers for generating `add` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_add (Type) ALUOp)
+(rule (aluop_add $I8) (ALUOp.Add32))
+(rule (aluop_add $I16) (ALUOp.Add32))
+(rule (aluop_add $I32) (ALUOp.Add32))
+(rule (aluop_add $I64) (ALUOp.Add64))
+
+(decl aluop_add_sext16 (Type) ALUOp)
+(rule (aluop_add_sext16 $I16) (ALUOp.Add32Ext16))
+(rule (aluop_add_sext16 $I32) (ALUOp.Add32Ext16))
+(rule (aluop_add_sext16 $I64) (ALUOp.Add64Ext16))
+
+(decl aluop_add_sext32 (Type) ALUOp)
+(rule (aluop_add_sext32 $I64) (ALUOp.Add64Ext32))
+
+(decl add_reg (Type Reg Reg) Reg)
+(rule (add_reg ty x y) (alu_rrr ty (aluop_add ty) x y))
+
+(decl add_reg_sext32 (Type Reg Reg) Reg)
+(rule (add_reg_sext32 ty x y) (alu_rr ty (aluop_add_sext32 ty) x y))
+
+(decl add_simm16 (Type Reg i16) Reg)
+(rule (add_simm16 ty x y) (alu_rrsimm16 ty (aluop_add ty) x y))
+
+(decl add_simm32 (Type Reg i32) Reg)
+(rule (add_simm32 ty x y) (alu_rsimm32 ty (aluop_add ty) x y))
+
+(decl add_mem (Type Reg MemArg) Reg)
+(rule (add_mem ty x y) (alu_rx ty (aluop_add ty) x y))
+
+(decl add_mem_sext16 (Type Reg MemArg) Reg)
+(rule (add_mem_sext16 ty x y) (alu_rx ty (aluop_add_sext16 ty) x y))
+
+(decl add_mem_sext32 (Type Reg MemArg) Reg)
+(rule (add_mem_sext32 ty x y) (alu_rx ty (aluop_add_sext32 ty) x y))
+
+
+;; Helpers for generating `add_logical` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_add_logical (Type) ALUOp)
+(rule (aluop_add_logical $I32) (ALUOp.AddLogical32))
+(rule (aluop_add_logical $I64) (ALUOp.AddLogical64))
+
+(decl aluop_add_logical_zext32 (Type) ALUOp)
+(rule (aluop_add_logical_zext32 $I64) (ALUOp.AddLogical64Ext32))
+
+(decl add_logical_reg (Type Reg Reg) Reg)
+(rule (add_logical_reg ty x y) (alu_rrr ty (aluop_add_logical ty) x y))
+
+(decl add_logical_reg_zext32 (Type Reg Reg) Reg)
+(rule (add_logical_reg_zext32 ty x y) (alu_rr ty (aluop_add_logical_zext32 ty) x y))
+
+(decl add_logical_zimm32 (Type Reg u32) Reg)
+(rule (add_logical_zimm32 ty x y) (alu_ruimm32 ty (aluop_add_logical ty) x y))
+
+(decl add_logical_mem (Type Reg MemArg) Reg)
+(rule (add_logical_mem ty x y) (alu_rx ty (aluop_add_logical ty) x y))
+
+(decl add_logical_mem_zext32 (Type Reg MemArg) Reg)
+(rule (add_logical_mem_zext32 ty x y) (alu_rx ty (aluop_add_logical_zext32 ty) x y))
+
+
+;; Helpers for generating `sub` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_sub (Type) ALUOp)
+(rule (aluop_sub $I8) (ALUOp.Sub32))
+(rule (aluop_sub $I16) (ALUOp.Sub32))
+(rule (aluop_sub $I32) (ALUOp.Sub32))
+(rule (aluop_sub $I64) (ALUOp.Sub64))
+
+(decl aluop_sub_sext16 (Type) ALUOp)
+(rule (aluop_sub_sext16 $I16) (ALUOp.Sub32Ext16))
+(rule (aluop_sub_sext16 $I32) (ALUOp.Sub32Ext16))
+(rule (aluop_sub_sext16 $I64) (ALUOp.Sub64Ext16))
+
+(decl aluop_sub_sext32 (Type) ALUOp)
+(rule (aluop_sub_sext32 $I64) (ALUOp.Sub64Ext32))
+
+(decl sub_reg (Type Reg Reg) Reg)
+(rule (sub_reg ty x y) (alu_rrr ty (aluop_sub ty) x y))
+
+(decl sub_reg_sext32 (Type Reg Reg) Reg)
+(rule (sub_reg_sext32 ty x y) (alu_rr ty (aluop_sub_sext32 ty) x y))
+
+(decl sub_mem (Type Reg MemArg) Reg)
+(rule (sub_mem ty x y) (alu_rx ty (aluop_sub ty) x y))
+
+(decl sub_mem_sext16 (Type Reg MemArg) Reg)
+(rule (sub_mem_sext16 ty x y) (alu_rx ty (aluop_sub_sext16 ty) x y))
+
+(decl sub_mem_sext32 (Type Reg MemArg) Reg)
+(rule (sub_mem_sext32 ty x y) (alu_rx ty (aluop_sub_sext32 ty) x y))
+
+
+;; Helpers for generating `sub_logical` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_sub_logical (Type) ALUOp)
+(rule (aluop_sub_logical $I32) (ALUOp.SubLogical32))
+(rule (aluop_sub_logical $I64) (ALUOp.SubLogical64))
+
+(decl aluop_sub_logical_zext32 (Type) ALUOp)
+(rule (aluop_sub_logical_zext32 $I64) (ALUOp.SubLogical64Ext32))
+
+(decl sub_logical_reg (Type Reg Reg) Reg)
+(rule (sub_logical_reg ty x y) (alu_rrr ty (aluop_sub_logical ty) x y))
+
+(decl sub_logical_reg_zext32 (Type Reg Reg) Reg)
+(rule (sub_logical_reg_zext32 ty x y) (alu_rr ty (aluop_sub_logical_zext32 ty) x y))
+
+(decl sub_logical_zimm32 (Type Reg u32) Reg)
+(rule (sub_logical_zimm32 ty x y) (alu_ruimm32 ty (aluop_sub_logical ty) x y))
+
+(decl sub_logical_mem (Type Reg MemArg) Reg)
+(rule (sub_logical_mem ty x y) (alu_rx ty (aluop_sub_logical ty) x y))
+
+(decl sub_logical_mem_zext32 (Type Reg MemArg) Reg)
+(rule (sub_logical_mem_zext32 ty x y) (alu_rx ty (aluop_sub_logical ty) x y))
+
+
+;; Helpers for generating `mul` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_mul (Type) ALUOp)
+(rule (aluop_mul $I8) (ALUOp.Mul32))
+(rule (aluop_mul $I16) (ALUOp.Mul32))
+(rule (aluop_mul $I32) (ALUOp.Mul32))
+(rule (aluop_mul $I64) (ALUOp.Mul64))
+
+(decl aluop_mul_sext16 (Type) ALUOp)
+(rule (aluop_mul_sext16 $I16) (ALUOp.Mul32Ext16))
+(rule (aluop_mul_sext16 $I32) (ALUOp.Mul32Ext16))
+(rule (aluop_mul_sext16 $I64) (ALUOp.Mul64Ext16))
+
+(decl aluop_mul_sext32 (Type) ALUOp)
+(rule (aluop_mul_sext32 $I64) (ALUOp.Mul64Ext32))
+
+(decl mul_reg (Type Reg Reg) Reg)
+(rule (mul_reg ty x y) (alu_rrr ty (aluop_mul ty) x y))
+
+(decl mul_reg_sext32 (Type Reg Reg) Reg)
+(rule (mul_reg_sext32 ty x y) (alu_rr ty (aluop_mul_sext32 ty) x y))
+
+(decl mul_simm16 (Type Reg i16) Reg)
+(rule (mul_simm16 ty x y) (alu_rsimm16 ty (aluop_mul ty) x y))
+
+(decl mul_simm32 (Type Reg i32) Reg)
+(rule (mul_simm32 ty x y) (alu_rsimm32 ty (aluop_mul ty) x y))
+
+(decl mul_mem (Type Reg MemArg) Reg)
+(rule (mul_mem ty x y) (alu_rx ty (aluop_mul ty) x y))
+
+(decl mul_mem_sext16 (Type Reg MemArg) Reg)
+(rule (mul_mem_sext16 ty x y) (alu_rx ty (aluop_mul_sext16 ty) x y))
+
+(decl mul_mem_sext32 (Type Reg MemArg) Reg)
+(rule (mul_mem_sext32 ty x y) (alu_rx ty (aluop_mul_sext32 ty) x y))
+
+
+;; Helpers for generating `udivmod` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl udivmod (Type RegPair Reg) RegPair)
+(rule (udivmod $I32 x y) (udivmod32 x y))
+(rule (udivmod $I64 x y) (udivmod64 x y))
+
+
+;; Helpers for generating `sdivmod` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl sdivmod (Type RegPair Reg) RegPair)
+(rule (sdivmod $I32 x y) (sdivmod32 x y))
+(rule (sdivmod $I64 x y) (sdivmod64 x y))
+
+
+;; Helpers for generating `and` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_and (Type) ALUOp)
+(rule (aluop_and (gpr32_ty _ty)) (ALUOp.And32))
+(rule (aluop_and (gpr64_ty _ty)) (ALUOp.And64))
+
+(decl and_reg (Type Reg Reg) Reg)
+(rule (and_reg ty x y) (alu_rrr ty (aluop_and ty) x y))
+
+(decl and_uimm16shifted (Type Reg UImm16Shifted) Reg)
+(rule (and_uimm16shifted ty x y) (alu_ruimm16shifted ty (aluop_and ty) x y))
+
+(decl and_uimm32shifted (Type Reg UImm32Shifted) Reg)
+(rule (and_uimm32shifted ty x y) (alu_ruimm32shifted ty (aluop_and ty) x y))
+
+(decl and_mem (Type Reg MemArg) Reg)
+(rule (and_mem ty x y) (alu_rx ty (aluop_and ty) x y))
+
+
+;; Helpers for generating `or` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_or (Type) ALUOp)
+(rule (aluop_or (gpr32_ty _ty)) (ALUOp.Orr32))
+(rule (aluop_or (gpr64_ty _ty)) (ALUOp.Orr64))
+
+(decl or_reg (Type Reg Reg) Reg)
+(rule (or_reg ty x y) (alu_rrr ty (aluop_or ty) x y))
+
+(decl or_uimm16shifted (Type Reg UImm16Shifted) Reg)
+(rule (or_uimm16shifted ty x y) (alu_ruimm16shifted ty (aluop_or ty) x y))
+
+(decl or_uimm32shifted (Type Reg UImm32Shifted) Reg)
+(rule (or_uimm32shifted ty x y) (alu_ruimm32shifted ty (aluop_or ty) x y))
+
+(decl or_mem (Type Reg MemArg) Reg)
+(rule (or_mem ty x y) (alu_rx ty (aluop_or ty) x y))
+
+
+;; Helpers for generating `xor` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_xor (Type) ALUOp)
+(rule (aluop_xor (gpr32_ty _ty)) (ALUOp.Xor32))
+(rule (aluop_xor (gpr64_ty _ty)) (ALUOp.Xor64))
+
+(decl xor_reg (Type Reg Reg) Reg)
+(rule (xor_reg ty x y) (alu_rrr ty (aluop_xor ty) x y))
+
+(decl xor_uimm32shifted (Type Reg UImm32Shifted) Reg)
+(rule (xor_uimm32shifted ty x y) (alu_ruimm32shifted ty (aluop_xor ty) x y))
+
+(decl xor_mem (Type Reg MemArg) Reg)
+(rule (xor_mem ty x y) (alu_rx ty (aluop_xor ty) x y))
+
+
+;; Helpers for generating `not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl not_reg (Type Reg) Reg)
+(rule (not_reg (gpr32_ty ty) x)
+      (xor_uimm32shifted ty x (uimm32shifted 0xffffffff 0)))
+(rule (not_reg (gpr64_ty ty) x)
+      (xor_uimm32shifted ty
+        (xor_uimm32shifted ty x (uimm32shifted 0xffffffff 0))
+        (uimm32shifted 0xffffffff 32)))
+
+
+;; Helpers for generating `and_not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_and_not (Type) ALUOp)
+(rule (aluop_and_not (gpr32_ty _ty)) (ALUOp.AndNot32))
+(rule (aluop_and_not (gpr64_ty _ty)) (ALUOp.AndNot64))
+
+(decl and_not_reg (Type Reg Reg) Reg)
+(rule (and_not_reg ty x y) (alu_rrr ty (aluop_and_not ty) x y))
+
+
+;; Helpers for generating `or_not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_or_not (Type) ALUOp)
+(rule (aluop_or_not (gpr32_ty _ty)) (ALUOp.OrrNot32))
+(rule (aluop_or_not (gpr64_ty _ty)) (ALUOp.OrrNot64))
+
+(decl or_not_reg (Type Reg Reg) Reg)
+(rule (or_not_reg ty x y) (alu_rrr ty (aluop_or_not ty) x y))
+
+
+;; Helpers for generating `xor_not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl aluop_xor_not (Type) ALUOp)
+(rule (aluop_xor_not (gpr32_ty _ty)) (ALUOp.XorNot32))
+(rule (aluop_xor_not (gpr64_ty _ty)) (ALUOp.XorNot64))
+
+(decl xor_not_reg (Type Reg Reg) Reg)
+(rule (xor_not_reg ty x y) (alu_rrr ty (aluop_xor_not ty) x y))
+
+
+;; Helpers for generating `abs` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl unaryop_abs (Type) UnaryOp)
+(rule (unaryop_abs $I32) (UnaryOp.Abs32))
+(rule (unaryop_abs $I64) (UnaryOp.Abs64))
+
+(decl unaryop_abs_sext32 (Type) UnaryOp)
+(rule (unaryop_abs_sext32 $I64) (UnaryOp.Abs64Ext32))
+
+(decl abs_reg (Type Reg) Reg)
+(rule (abs_reg ty x) (unary_rr ty (unaryop_abs ty) x))
+
+(decl abs_reg_sext32 (Type Reg) Reg)
+(rule (abs_reg_sext32 ty x) (unary_rr ty (unaryop_abs_sext32 ty) x))
+
+
+;; Helpers for generating `neg` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl unaryop_neg (Type) UnaryOp)
+(rule (unaryop_neg $I8) (UnaryOp.Neg32))
+(rule (unaryop_neg $I16) (UnaryOp.Neg32))
+(rule (unaryop_neg $I32) (UnaryOp.Neg32))
+(rule (unaryop_neg $I64) (UnaryOp.Neg64))
+
+(decl unaryop_neg_sext32 (Type) UnaryOp)
+(rule (unaryop_neg_sext32 $I64) (UnaryOp.Neg64Ext32))
+
+(decl neg_reg (Type Reg) Reg)
+(rule (neg_reg ty x) (unary_rr ty (unaryop_neg ty) x))
+
+(decl neg_reg_sext32 (Type Reg) Reg)
+(rule (neg_reg_sext32 ty x) (unary_rr ty (unaryop_neg_sext32 ty) x))
+
+
+;; Helpers for generating `rot` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl shiftop_rot (Type) ShiftOp)
+(rule (shiftop_rot $I32) (ShiftOp.RotL32))
+(rule (shiftop_rot $I64) (ShiftOp.RotL64))
+
+(decl rot_reg (Type Reg Reg) Reg)
+(rule (rot_reg ty x shift_reg)
+      (shift_rr ty (shiftop_rot ty) x 0 shift_reg))
+
+(decl rot_imm (Type Reg u8) Reg)
+(rule (rot_imm ty x shift_imm)
+      (shift_rr ty (shiftop_rot ty) x shift_imm (zero_reg)))
+
+
+;; Helpers for generating `lshl` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl shiftop_lshl (Type) ShiftOp)
+(rule (shiftop_lshl $I8) (ShiftOp.LShL32))
+(rule (shiftop_lshl $I16) (ShiftOp.LShL32))
+(rule (shiftop_lshl $I32) (ShiftOp.LShL32))
+(rule (shiftop_lshl $I64) (ShiftOp.LShL64))
+
+(decl lshl_reg (Type Reg Reg) Reg)
+(rule (lshl_reg ty x shift_reg)
+      (shift_rr ty (shiftop_lshl ty) x 0 shift_reg))
+
+(decl lshl_imm (Type Reg u8) Reg)
+(rule (lshl_imm ty x shift_imm)
+      (shift_rr ty (shiftop_lshl ty) x shift_imm (zero_reg)))
+
+
+;; Helpers for generating `lshr` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl shiftop_lshr (Type) ShiftOp)
+(rule (shiftop_lshr $I32) (ShiftOp.LShR32))
+(rule (shiftop_lshr $I64) (ShiftOp.LShR64))
+
+(decl lshr_reg (Type Reg Reg) Reg)
+(rule (lshr_reg ty x shift_reg)
+      (shift_rr ty (shiftop_lshr ty) x 0 shift_reg))
+
+(decl lshr_imm (Type Reg u8) Reg)
+(rule (lshr_imm ty x shift_imm)
+      (shift_rr ty (shiftop_lshr ty) x shift_imm (zero_reg)))
+
+
+;; Helpers for generating `ashr` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl shiftop_ashr (Type) ShiftOp)
+(rule (shiftop_ashr $I32) (ShiftOp.AShR32))
+(rule (shiftop_ashr $I64) (ShiftOp.AShR64))
+
+(decl ashr_reg (Type Reg Reg) Reg)
+(rule (ashr_reg ty x shift_reg)
+      (shift_rr ty (shiftop_ashr ty) x 0 shift_reg))
+
+(decl ashr_imm (Type Reg u8) Reg)
+(rule (ashr_imm ty x shift_imm)
+      (shift_rr ty (shiftop_ashr ty) x shift_imm (zero_reg)))
+
+
+;; Helpers for generating `popcnt` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl popcnt_byte (Reg) Reg)
+(rule (popcnt_byte x) (unary_rr $I64 (UnaryOp.PopcntByte) x))
+
+(decl popcnt_reg (Reg) Reg)
+(rule (popcnt_reg x) (unary_rr $I64 (UnaryOp.PopcntReg) x))
+
+
+;; Helpers for generating `atomic_rmw` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl atomic_rmw_and (Type Reg MemArg) Reg)
+(rule (atomic_rmw_and $I32 src mem) (atomic_rmw_impl $I32 (ALUOp.And32) src mem))
+(rule (atomic_rmw_and $I64 src mem) (atomic_rmw_impl $I64 (ALUOp.And64) src mem))
+
+(decl atomic_rmw_or (Type Reg MemArg) Reg)
+(rule (atomic_rmw_or $I32 src mem) (atomic_rmw_impl $I32 (ALUOp.Orr32) src mem))
+(rule (atomic_rmw_or $I64 src mem) (atomic_rmw_impl $I64 (ALUOp.Orr64) src mem))
+
+(decl atomic_rmw_xor (Type Reg MemArg) Reg)
+(rule (atomic_rmw_xor $I32 src mem) (atomic_rmw_impl $I32 (ALUOp.Xor32) src mem))
+(rule (atomic_rmw_xor $I64 src mem) (atomic_rmw_impl $I64 (ALUOp.Xor64) src mem))
+
+(decl atomic_rmw_add (Type Reg MemArg) Reg)
+(rule (atomic_rmw_add $I32 src mem) (atomic_rmw_impl $I32 (ALUOp.Add32) src mem))
+(rule (atomic_rmw_add $I64 src mem) (atomic_rmw_impl $I64 (ALUOp.Add64) src mem))
+
+
+;; Helpers for generating `fadd` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_add (Type) FPUOp2)
+(rule (fpuop2_add $F32) (FPUOp2.Add32))
+(rule (fpuop2_add $F64) (FPUOp2.Add64))
+
+(decl fadd_reg (Type Reg Reg) Reg)
+(rule (fadd_reg ty x y) (fpu_rrr ty (fpuop2_add ty) x y))
+
+
+;; Helpers for generating `fsub` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_sub (Type) FPUOp2)
+(rule (fpuop2_sub $F32) (FPUOp2.Sub32))
+(rule (fpuop2_sub $F64) (FPUOp2.Sub64))
+
+(decl fsub_reg (Type Reg Reg) Reg)
+(rule (fsub_reg ty x y) (fpu_rrr ty (fpuop2_sub ty) x y))
+
+
+;; Helpers for generating `fmul` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_mul (Type) FPUOp2)
+(rule (fpuop2_mul $F32) (FPUOp2.Mul32))
+(rule (fpuop2_mul $F64) (FPUOp2.Mul64))
+
+(decl fmul_reg (Type Reg Reg) Reg)
+(rule (fmul_reg ty x y) (fpu_rrr ty (fpuop2_mul ty) x y))
+
+
+;; Helpers for generating `fdiv` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_div (Type) FPUOp2)
+(rule (fpuop2_div $F32) (FPUOp2.Div32))
+(rule (fpuop2_div $F64) (FPUOp2.Div64))
+
+(decl fdiv_reg (Type Reg Reg) Reg)
+(rule (fdiv_reg ty x y) (fpu_rrr ty (fpuop2_div ty) x y))
+
+
+;; Helpers for generating `fmin` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_min (Type) FPUOp2)
+(rule (fpuop2_min $F32) (FPUOp2.Min32))
+(rule (fpuop2_min $F64) (FPUOp2.Min64))
+
+(decl fmin_reg (Type Reg Reg) Reg)
+(rule (fmin_reg ty x y) (fpuvec_rrr ty (fpuop2_min ty) x y))
+
+
+;; Helpers for generating `fmax` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_max (Type) FPUOp2)
+(rule (fpuop2_max $F32) (FPUOp2.Max32))
+(rule (fpuop2_max $F64) (FPUOp2.Max64))
+
+(decl fmax_reg (Type Reg Reg) Reg)
+(rule (fmax_reg ty x y) (fpuvec_rrr ty (fpuop2_max ty) x y))
+
+
+;; Helpers for generating `fma` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop3_fma (Type) FPUOp3)
+(rule (fpuop3_fma $F32) (FPUOp3.MAdd32))
+(rule (fpuop3_fma $F64) (FPUOp3.MAdd64))
+
+(decl fma_reg (Type Reg Reg Reg) Reg)
+(rule (fma_reg ty x y acc) (fpu_rrrr ty (fpuop3_fma ty) acc x y))
+
+
+;; Helpers for generating `sqrt` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop1_sqrt (Type) FPUOp1)
+(rule (fpuop1_sqrt $F32) (FPUOp1.Sqrt32))
+(rule (fpuop1_sqrt $F64) (FPUOp1.Sqrt64))
+
+(decl sqrt_reg (Type Reg) Reg)
+(rule (sqrt_reg ty x) (fpu_rr ty (fpuop1_sqrt ty) x))
+
+
+;; Helpers for generating `fneg` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop1_neg (Type) FPUOp1)
+(rule (fpuop1_neg $F32) (FPUOp1.Neg32))
+(rule (fpuop1_neg $F64) (FPUOp1.Neg64))
+
+(decl fneg_reg (Type Reg) Reg)
+(rule (fneg_reg ty x) (fpu_rr ty (fpuop1_neg ty) x))
+
+
+;; Helpers for generating `fabs` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop1_abs (Type) FPUOp1)
+(rule (fpuop1_abs $F32) (FPUOp1.Abs32))
+(rule (fpuop1_abs $F64) (FPUOp1.Abs64))
+
+(decl fabs_reg (Type Reg) Reg)
+(rule (fabs_reg ty x) (fpu_rr ty (fpuop1_abs ty) x))
+
+
+;; Helpers for generating `ceil` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuroundmode_ceil (Type) FpuRoundMode)
+(rule (fpuroundmode_ceil $F32) (FpuRoundMode.Plus32))
+(rule (fpuroundmode_ceil $F64) (FpuRoundMode.Plus64))
+
+(decl ceil_reg (Type Reg) Reg)
+(rule (ceil_reg ty x) (fpu_round ty (fpuroundmode_ceil ty) x))
+
+
+;; Helpers for generating `floor` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuroundmode_floor (Type) FpuRoundMode)
+(rule (fpuroundmode_floor $F32) (FpuRoundMode.Minus32))
+(rule (fpuroundmode_floor $F64) (FpuRoundMode.Minus64))
+
+(decl floor_reg (Type Reg) Reg)
+(rule (floor_reg ty x) (fpu_round ty (fpuroundmode_floor ty) x))
+
+
+;; Helpers for generating `trunc` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuroundmode_trunc (Type) FpuRoundMode)
+(rule (fpuroundmode_trunc $F32) (FpuRoundMode.Zero32))
+(rule (fpuroundmode_trunc $F64) (FpuRoundMode.Zero64))
+
+(decl trunc_reg (Type Reg) Reg)
+(rule (trunc_reg ty x) (fpu_round ty (fpuroundmode_trunc ty) x))
+
+
+;; Helpers for generating `nearest` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuroundmode_nearest (Type) FpuRoundMode)
+(rule (fpuroundmode_nearest $F32) (FpuRoundMode.Nearest32))
+(rule (fpuroundmode_nearest $F64) (FpuRoundMode.Nearest64))
+
+(decl nearest_reg (Type Reg) Reg)
+(rule (nearest_reg ty x) (fpu_round ty (fpuroundmode_nearest ty) x))
+
+
+;; Helpers for generating `fpromote` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop1_promote (Type Type) FPUOp1)
+(rule (fpuop1_promote $F64 $F32) (FPUOp1.Cvt32To64))
+
+(decl fpromote_reg (Type Type Reg) Reg)
+(rule (fpromote_reg dst_ty src_ty x)
+      (fpu_rr dst_ty (fpuop1_promote dst_ty src_ty) x))
+
+
+;; Helpers for generating `fdemote` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop1_demote (Type Type) FPUOp1)
+(rule (fpuop1_demote $F32 $F64) (FPUOp1.Cvt64To32))
+
+(decl fdemote_reg (Type Type Reg) Reg)
+(rule (fdemote_reg dst_ty src_ty x)
+      (fpu_rr dst_ty (fpuop1_demote dst_ty src_ty) x))
+
+
+;; Helpers for generating `fcvt_from_uint` instructions ;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl uint_to_fpu_op (Type Type) IntToFpuOp)
+(rule (uint_to_fpu_op $F32 $I32) (IntToFpuOp.U32ToF32))
+(rule (uint_to_fpu_op $F64 $I32) (IntToFpuOp.U32ToF64))
+(rule (uint_to_fpu_op $F32 $I64) (IntToFpuOp.U64ToF32))
+(rule (uint_to_fpu_op $F64 $I64) (IntToFpuOp.U64ToF64))
+
+(decl fcvt_from_uint_reg (Type Type Reg) Reg)
+(rule (fcvt_from_uint_reg dst_ty src_ty x)
+      (int_to_fpu dst_ty (uint_to_fpu_op dst_ty src_ty) x))
+
+
+;; Helpers for generating `fcvt_from_sint` instructions ;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl sint_to_fpu_op (Type Type) IntToFpuOp)
+(rule (sint_to_fpu_op $F32 $I32) (IntToFpuOp.I32ToF32))
+(rule (sint_to_fpu_op $F64 $I32) (IntToFpuOp.I32ToF64))
+(rule (sint_to_fpu_op $F32 $I64) (IntToFpuOp.I64ToF32))
+(rule (sint_to_fpu_op $F64 $I64) (IntToFpuOp.I64ToF64))
+
+(decl fcvt_from_sint_reg (Type Type Reg) Reg)
+(rule (fcvt_from_sint_reg dst_ty src_ty x)
+      (int_to_fpu dst_ty (sint_to_fpu_op dst_ty src_ty) x))
+
+
+;; Helpers for generating `fcvt_to_uint` instructions ;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpu_to_uint_op (Type Type) FpuToIntOp)
+(rule (fpu_to_uint_op $I32 $F32) (FpuToIntOp.F32ToU32))
+(rule (fpu_to_uint_op $I32 $F64) (FpuToIntOp.F64ToU32))
+(rule (fpu_to_uint_op $I64 $F32) (FpuToIntOp.F32ToU64))
+(rule (fpu_to_uint_op $I64 $F64) (FpuToIntOp.F64ToU64))
+
+(decl fcvt_to_uint_reg_with_flags (Type Type Reg) ProducesFlags)
+(rule (fcvt_to_uint_reg_with_flags dst_ty src_ty x)
+      (fpu_to_int dst_ty (fpu_to_uint_op dst_ty src_ty) x))
+
+(decl fcvt_to_uint_reg (Type Type Reg) Reg)
+(rule (fcvt_to_uint_reg dst_ty src_ty x)
+      (drop_flags (fcvt_to_uint_reg_with_flags dst_ty src_ty x)))
+
+
+;; Helpers for generating `fcvt_to_sint` instructions ;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpu_to_sint_op (Type Type) FpuToIntOp)
+(rule (fpu_to_sint_op $I32 $F32) (FpuToIntOp.F32ToI32))
+(rule (fpu_to_sint_op $I32 $F64) (FpuToIntOp.F64ToI32))
+(rule (fpu_to_sint_op $I64 $F32) (FpuToIntOp.F32ToI64))
+(rule (fpu_to_sint_op $I64 $F64) (FpuToIntOp.F64ToI64))
+
+(decl fcvt_to_sint_reg_with_flags (Type Type Reg) ProducesFlags)
+(rule (fcvt_to_sint_reg_with_flags dst_ty src_ty x)
+      (fpu_to_int dst_ty (fpu_to_sint_op dst_ty src_ty) x))
+
+(decl fcvt_to_sint_reg (Type Type Reg) Reg)
+(rule (fcvt_to_sint_reg dst_ty src_ty x)
+      (drop_flags (fcvt_to_sint_reg_with_flags dst_ty src_ty x)))
+
+
+;; Helpers for generating signed `icmp` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl cmpop_cmps (Type) CmpOp)
+(rule (cmpop_cmps $I32) (CmpOp.CmpS32))
+(rule (cmpop_cmps $I64) (CmpOp.CmpS64))
+
+(decl cmpop_cmps_sext16 (Type) CmpOp)
+(rule (cmpop_cmps_sext16 $I32) (CmpOp.CmpS32Ext16))
+(rule (cmpop_cmps_sext16 $I64) (CmpOp.CmpS64Ext16))
+
+(decl cmpop_cmps_sext32 (Type) CmpOp)
+(rule (cmpop_cmps_sext32 $I64) (CmpOp.CmpS64Ext32))
+
+(decl icmps_reg (Type Reg Reg) ProducesFlags)
+(rule (icmps_reg ty src1 src2) (cmp_rr (cmpop_cmps ty) src1 src2))
+
+(decl icmps_reg_sext32 (Type Reg Reg) ProducesFlags)
+(rule (icmps_reg_sext32 ty src1 src2) (cmp_rr (cmpop_cmps_sext32 ty) src1 src2))
+
+(decl icmps_simm16 (Type Reg i16) ProducesFlags)
+(rule (icmps_simm16 ty src imm) (cmp_rsimm16 (cmpop_cmps ty) src imm))
+
+(decl icmps_simm32 (Type Reg i32) ProducesFlags)
+(rule (icmps_simm32 ty src imm) (cmp_rsimm32 (cmpop_cmps ty) src imm))
+
+(decl icmps_mem (Type Reg MemArg) ProducesFlags)
+(rule (icmps_mem ty src mem) (cmp_rx (cmpop_cmps ty) src mem))
+
+(decl icmps_mem_sext16 (Type Reg MemArg) ProducesFlags)
+(rule (icmps_mem_sext16 ty src mem) (cmp_rx (cmpop_cmps_sext16 ty) src mem))
+
+(decl icmps_mem_sext32 (Type Reg MemArg) ProducesFlags)
+(rule (icmps_mem_sext32 ty src mem) (cmp_rx (cmpop_cmps_sext32 ty) src mem))
+
+
+;; Helpers for generating unsigned `icmp` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl cmpop_cmpu (Type) CmpOp)
+(rule (cmpop_cmpu $I32) (CmpOp.CmpL32))
+(rule (cmpop_cmpu $I64) (CmpOp.CmpL64))
+
+(decl cmpop_cmpu_zext16 (Type) CmpOp)
+(rule (cmpop_cmpu_zext16 $I32) (CmpOp.CmpL32Ext16))
+(rule (cmpop_cmpu_zext16 $I64) (CmpOp.CmpL64Ext16))
+
+(decl cmpop_cmpu_zext32 (Type) CmpOp)
+(rule (cmpop_cmpu_zext32 $I64) (CmpOp.CmpL64Ext32))
+
+(decl icmpu_reg (Type Reg Reg) ProducesFlags)
+(rule (icmpu_reg ty src1 src2) (cmp_rr (cmpop_cmpu ty) src1 src2))
+
+(decl icmpu_reg_zext32 (Type Reg Reg) ProducesFlags)
+(rule (icmpu_reg_zext32 ty src1 src2) (cmp_rr (cmpop_cmpu_zext32 ty) src1 src2))
+
+(decl icmpu_uimm32 (Type Reg u32) ProducesFlags)
+(rule (icmpu_uimm32 ty src imm) (cmp_ruimm32 (cmpop_cmpu ty) src imm))
+
+(decl icmpu_mem (Type Reg MemArg) ProducesFlags)
+(rule (icmpu_mem ty src mem) (cmp_rx (cmpop_cmpu ty) src mem))
+
+(decl icmpu_mem_zext16 (Type Reg MemArg) ProducesFlags)
+(rule (icmpu_mem_zext16 ty src mem) (cmp_rx (cmpop_cmpu_zext16 ty) src mem))
+
+(decl icmpu_mem_zext32 (Type Reg MemArg) ProducesFlags)
+(rule (icmpu_mem_zext32 ty src mem) (cmp_rx (cmpop_cmpu_zext32 ty) src mem))
+
+
+;; Helpers for generating `fcmp` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fcmp_reg (Type Reg Reg) ProducesFlags)
+(rule (fcmp_reg $F32 src1 src2) (fpu_cmp32 src1 src2))
+(rule (fcmp_reg $F64 src1 src2) (fpu_cmp64 src1 src2))
+
+
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 4ee366482af8..c3f5f642a029 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -4,8 +4,7 @@
 #![allow(dead_code)]
 
 use crate::binemit::{Addend, CodeOffset, Reloc};
-use crate::ir::{types, ExternalName, Opcode, TrapCode, Type, ValueLabel};
-use crate::isa::unwind::UnwindInst;
+use crate::ir::{types, ExternalName, Opcode, Type, ValueLabel};
 use crate::machinst::*;
 use crate::{settings, CodegenError, CodegenResult};
 
@@ -34,199 +33,10 @@ mod emit_tests;
 //=============================================================================
 // Instructions (top level): definition
 
-/// Supported instruction sets
-#[allow(non_camel_case_types)]
-#[derive(Debug)]
-pub(crate) enum InstructionSet {
-    /// Baseline ISA for cranelift is z14.
-    Base,
-    /// Miscellaneous-Instruction-Extensions Facility 2 (z15)
-    MIE2,
-    /// Vector-Enhancements Facility 2 (z15)
-    VXRS_EXT2,
-}
-
-/// An ALU operation. This can be paired with several instruction formats
-/// below (see `Inst`) in any combination.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum ALUOp {
-    Add32,
-    Add32Ext16,
-    Add64,
-    Add64Ext16,
-    Add64Ext32,
-    AddLogical32,
-    AddLogical64,
-    AddLogical64Ext32,
-    Sub32,
-    Sub32Ext16,
-    Sub64,
-    Sub64Ext16,
-    Sub64Ext32,
-    SubLogical32,
-    SubLogical64,
-    SubLogical64Ext32,
-    Mul32,
-    Mul32Ext16,
-    Mul64,
-    Mul64Ext16,
-    Mul64Ext32,
-    And32,
-    And64,
-    Orr32,
-    Orr64,
-    Xor32,
-    Xor64,
-    /// NAND
-    AndNot32,
-    AndNot64,
-    /// NOR
-    OrrNot32,
-    OrrNot64,
-    /// XNOR
-    XorNot32,
-    XorNot64,
-}
-
-impl ALUOp {
-    pub(crate) fn available_from(&self) -> InstructionSet {
-        match self {
-            ALUOp::AndNot32 | ALUOp::AndNot64 => InstructionSet::MIE2,
-            ALUOp::OrrNot32 | ALUOp::OrrNot64 => InstructionSet::MIE2,
-            ALUOp::XorNot32 | ALUOp::XorNot64 => InstructionSet::MIE2,
-            _ => InstructionSet::Base,
-        }
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum UnaryOp {
-    Abs32,
-    Abs64,
-    Abs64Ext32,
-    Neg32,
-    Neg64,
-    Neg64Ext32,
-    PopcntByte,
-    PopcntReg,
-}
-
-impl UnaryOp {
-    pub(crate) fn available_from(&self) -> InstructionSet {
-        match self {
-            UnaryOp::PopcntReg => InstructionSet::MIE2,
-            _ => InstructionSet::Base,
-        }
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum ShiftOp {
-    RotL32,
-    RotL64,
-    LShL32,
-    LShL64,
-    LShR32,
-    LShR64,
-    AShR32,
-    AShR64,
-}
-
-/// An integer comparison operation.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum CmpOp {
-    CmpS32,
-    CmpS32Ext16,
-    CmpS64,
-    CmpS64Ext16,
-    CmpS64Ext32,
-    CmpL32,
-    CmpL32Ext16,
-    CmpL64,
-    CmpL64Ext16,
-    CmpL64Ext32,
-}
-
-/// A floating-point unit (FPU) operation with one arg.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum FPUOp1 {
-    Abs32,
-    Abs64,
-    Neg32,
-    Neg64,
-    NegAbs32,
-    NegAbs64,
-    Sqrt32,
-    Sqrt64,
-    Cvt32To64,
-    Cvt64To32,
-}
-
-/// A floating-point unit (FPU) operation with two args.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum FPUOp2 {
-    Add32,
-    Add64,
-    Sub32,
-    Sub64,
-    Mul32,
-    Mul64,
-    Div32,
-    Div64,
-    Max32,
-    Max64,
-    Min32,
-    Min64,
-}
-
-/// A floating-point unit (FPU) operation with three args.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum FPUOp3 {
-    MAdd32,
-    MAdd64,
-    MSub32,
-    MSub64,
-}
-
-/// A conversion from an FP to an integer value.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum FpuToIntOp {
-    F32ToU32,
-    F32ToI32,
-    F32ToU64,
-    F32ToI64,
-    F64ToU32,
-    F64ToI32,
-    F64ToU64,
-    F64ToI64,
-}
-
-/// A conversion from an integer to an FP value.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum IntToFpuOp {
-    U32ToF32,
-    I32ToF32,
-    U32ToF64,
-    I32ToF64,
-    U64ToF32,
-    I64ToF32,
-    U64ToF64,
-    I64ToF64,
-}
-
-/// Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to
-/// nearest, and for 32- or 64-bit FP values.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub enum FpuRoundMode {
-    Minus32,
-    Minus64,
-    Plus32,
-    Plus64,
-    Zero32,
-    Zero64,
-    Nearest32,
-    Nearest64,
-}
+pub use crate::isa::s390x::lower::isle::generated_code::{
+    ALUOp, CmpOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst,
+    ShiftOp, UnaryOp,
+};
 
 /// Additional information for (direct) Call instructions, left out of line to lower the size of
 /// the Inst enum.
@@ -257,726 +67,6 @@ pub struct JTSequenceInfo {
     pub targets_for_term: Vec<MachLabel>, // needed for MachTerminator.
 }
 
-/// Instruction formats.
-#[derive(Clone, Debug)]
-pub enum Inst {
-    /// A no-op of zero size.
-    Nop0,
-
-    /// A no-op of size two bytes.
-    Nop2,
-
-    /// An ALU operation with two register sources and a register destination.
-    AluRRR {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        rn: Reg,
-        rm: Reg,
-    },
-    /// An ALU operation with a register source and a signed 16-bit
-    /// immediate source, and a separate register destination.
-    AluRRSImm16 {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        rn: Reg,
-        imm: i16,
-    },
-    /// An ALU operation with a register in-/out operand and
-    /// a second register source.
-    AluRR {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        rm: Reg,
-    },
-    /// An ALU operation with a register in-/out operand and
-    /// a memory source.
-    AluRX {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// An ALU operation with a register in-/out operand and a signed 16-bit
-    /// immediate source.
-    AluRSImm16 {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        imm: i16,
-    },
-    /// An ALU operation with a register in-/out operand and a signed 32-bit
-    /// immediate source.
-    AluRSImm32 {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        imm: i32,
-    },
-    /// An ALU operation with a register in-/out operand and an unsigned 32-bit
-    /// immediate source.
-    AluRUImm32 {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        imm: u32,
-    },
-    /// An ALU operation with a register in-/out operand and a shifted 16-bit
-    /// immediate source.
-    AluRUImm16Shifted {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        imm: UImm16Shifted,
-    },
-    /// An ALU operation with a register in-/out operand and a shifted 32-bit
-    /// immediate source.
-    AluRUImm32Shifted {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        imm: UImm32Shifted,
-    },
-    /// A multiply operation with two register sources and a register pair destination.
-    /// FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
-    SMulWide {
-        rn: Reg,
-        rm: Reg,
-    },
-    /// A multiply operation with an in/out register pair, and an extra register source.
-    /// Only the lower half of the register pair is used as input.
-    /// FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
-    UMulWide {
-        rn: Reg,
-    },
-    /// A divide operation with an in/out register pair, and an extra register source.
-    /// Only the lower half of the register pair is used as input.
-    /// FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
-    SDivMod32 {
-        rn: Reg,
-    },
-    SDivMod64 {
-        rn: Reg,
-    },
-    /// A divide operation with an in/out register pair, and an extra register source.
-    /// FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
-    UDivMod32 {
-        rn: Reg,
-    },
-    UDivMod64 {
-        rn: Reg,
-    },
-    /// A FLOGR operation with a register source and a register pair destination.
-    /// FIXME: The pair is hard-coded as %r0/%r1 because regalloc cannot handle pairs.
-    Flogr {
-        rn: Reg,
-    },
-
-    /// A shift instruction with a register source, a register destination,
-    /// and an immediate plus an optional register as shift count.
-    ShiftRR {
-        shift_op: ShiftOp,
-        rd: Writable<Reg>,
-        rn: Reg,
-        shift_imm: u8,
-        shift_reg: Reg,
-    },
-
-    /// An unary operation with a register source and a register destination.
-    UnaryRR {
-        op: UnaryOp,
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-
-    /// A compare operation with two register sources.
-    CmpRR {
-        op: CmpOp,
-        rn: Reg,
-        rm: Reg,
-    },
-    /// A compare operation with a register source and a memory source.
-    CmpRX {
-        op: CmpOp,
-        rn: Reg,
-        mem: MemArg,
-    },
-    /// A compare operation with a register source and a signed 16-bit
-    /// immediate source.
-    CmpRSImm16 {
-        op: CmpOp,
-        rn: Reg,
-        imm: i16,
-    },
-    /// A compare operation with a register source and a signed 32-bit
-    /// immediate source.
-    CmpRSImm32 {
-        op: CmpOp,
-        rn: Reg,
-        imm: i32,
-    },
-    /// A compare operation with a register source and a unsigned 32-bit
-    /// immediate source.
-    CmpRUImm32 {
-        op: CmpOp,
-        rn: Reg,
-        imm: u32,
-    },
-    /// A compare-and-trap instruction with two register sources.
-    CmpTrapRR {
-        op: CmpOp,
-        rn: Reg,
-        rm: Reg,
-        cond: Cond,
-        trap_code: TrapCode,
-    },
-    /// A compare-and-trap operation with a register source and a signed 16-bit
-    /// immediate source.
-    CmpTrapRSImm16 {
-        op: CmpOp,
-        rn: Reg,
-        imm: i16,
-        cond: Cond,
-        trap_code: TrapCode,
-    },
-    /// A compare-and-trap operation with a register source and an unsigned 16-bit
-    /// immediate source.
-    CmpTrapRUImm16 {
-        op: CmpOp,
-        rn: Reg,
-        imm: u16,
-        cond: Cond,
-        trap_code: TrapCode,
-    },
-
-    /// An atomic read-modify-write operation with a memory in-/out operand,
-    /// a register destination, and a register source.
-    /// a memory source.
-    AtomicRmw {
-        alu_op: ALUOp,
-        rd: Writable<Reg>,
-        rn: Reg,
-        mem: MemArg,
-    },
-    /// A 32-bit atomic compare-and-swap operation.
-    AtomicCas32 {
-        rd: Writable<Reg>,
-        rn: Reg,
-        mem: MemArg,
-    },
-    /// A 64-bit atomic compare-and-swap operation.
-    AtomicCas64 {
-        rd: Writable<Reg>,
-        rn: Reg,
-        mem: MemArg,
-    },
-    /// A memory fence operation.
-    Fence,
-
-    /// A 32-bit load.
-    Load32 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// An unsigned (zero-extending) 8-bit to 32-bit load.
-    Load32ZExt8 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A signed (sign-extending) 8-bit to 32-bit load.
-    Load32SExt8 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// An unsigned (zero-extending) 16-bit to 32-bit load.
-    Load32ZExt16 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A signed (sign-extending) 16-bit to 32-bit load.
-    Load32SExt16 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A 64-bit load.
-    Load64 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// An unsigned (zero-extending) 8-bit to 64-bit load.
-    Load64ZExt8 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A signed (sign-extending) 8-bit to 64-bit load.
-    Load64SExt8 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// An unsigned (zero-extending) 16-bit to 64-bit load.
-    Load64ZExt16 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A signed (sign-extending) 16-bit to 64-bit load.
-    Load64SExt16 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// An unsigned (zero-extending) 32-bit to 64-bit load.
-    Load64ZExt32 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A signed (sign-extending) 32-bit to 64-bit load.
-    Load64SExt32 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-
-    /// A 16-bit byte-reversed load.
-    LoadRev16 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A 32-bit byte-reversed load.
-    LoadRev32 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A 64-bit byte-reversed load.
-    LoadRev64 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-
-    /// An 8-bit store.
-    Store8 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// A 16-bit store.
-    Store16 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// A 32-bit store.
-    Store32 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// A 64-bit store.
-    Store64 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// An 8-bit store of an immediate.
-    StoreImm8 {
-        imm: u8,
-        mem: MemArg,
-    },
-    /// A 16-bit store of an immediate.
-    StoreImm16 {
-        imm: i16,
-        mem: MemArg,
-    },
-    /// A 32-bit store of a sign-extended 16-bit immediate.
-    StoreImm32SExt16 {
-        imm: i16,
-        mem: MemArg,
-    },
-    /// A 64-bit store of a sign-extended 16-bit immediate.
-    StoreImm64SExt16 {
-        imm: i16,
-        mem: MemArg,
-    },
-
-    /// A 16-bit byte-reversed store.
-    StoreRev16 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// A 32-bit byte-reversed store.
-    StoreRev32 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// A 64-bit byte-reversed store.
-    StoreRev64 {
-        rd: Reg,
-        mem: MemArg,
-    },
-
-    /// A load-multiple instruction.
-    LoadMultiple64 {
-        rt: Writable<Reg>,
-        rt2: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// A store-multiple instruction.
-    StoreMultiple64 {
-        rt: Reg,
-        rt2: Reg,
-        mem: MemArg,
-    },
-
-    /// A 32-bit move instruction.
-    Mov32 {
-        rd: Writable<Reg>,
-        rm: Reg,
-    },
-    /// A 64-bit move instruction.
-    Mov64 {
-        rd: Writable<Reg>,
-        rm: Reg,
-    },
-    /// A 32-bit move instruction with a full 32-bit immediate.
-    Mov32Imm {
-        rd: Writable<Reg>,
-        imm: u32,
-    },
-    /// A 32-bit move instruction with a 16-bit signed immediate.
-    Mov32SImm16 {
-        rd: Writable<Reg>,
-        imm: i16,
-    },
-    /// A 64-bit move instruction with a 16-bit signed immediate.
-    Mov64SImm16 {
-        rd: Writable<Reg>,
-        imm: i16,
-    },
-    /// A 64-bit move instruction with a 32-bit signed immediate.
-    Mov64SImm32 {
-        rd: Writable<Reg>,
-        imm: i32,
-    },
-    /// A 64-bit move instruction with a shifted 16-bit immediate.
-    Mov64UImm16Shifted {
-        rd: Writable<Reg>,
-        imm: UImm16Shifted,
-    },
-    /// A 64-bit move instruction with a shifted 32-bit immediate.
-    Mov64UImm32Shifted {
-        rd: Writable<Reg>,
-        imm: UImm32Shifted,
-    },
-
-    /// A 64-bit insert instruction with a shifted 16-bit immediate.
-    Insert64UImm16Shifted {
-        rd: Writable<Reg>,
-        imm: UImm16Shifted,
-    },
-    /// A 64-bit insert instruction with a shifted 32-bit immediate.
-    Insert64UImm32Shifted {
-        rd: Writable<Reg>,
-        imm: UImm32Shifted,
-    },
-
-    /// A sign- or zero-extend operation.
-    Extend {
-        rd: Writable<Reg>,
-        rn: Reg,
-        signed: bool,
-        from_bits: u8,
-        to_bits: u8,
-    },
-
-    /// A 32-bit conditional move instruction.
-    CMov32 {
-        rd: Writable<Reg>,
-        cond: Cond,
-        rm: Reg,
-    },
-    /// A 64-bit conditional move instruction.
-    CMov64 {
-        rd: Writable<Reg>,
-        cond: Cond,
-        rm: Reg,
-    },
-    /// A 32-bit conditional move instruction with a 16-bit signed immediate.
-    CMov32SImm16 {
-        rd: Writable<Reg>,
-        cond: Cond,
-        imm: i16,
-    },
-    /// A 64-bit conditional move instruction with a 16-bit signed immediate.
-    CMov64SImm16 {
-        rd: Writable<Reg>,
-        cond: Cond,
-        imm: i16,
-    },
-
-    /// 32-bit FPU move.
-    FpuMove32 {
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-    /// 64-bit FPU move.
-    FpuMove64 {
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-
-    /// A 32-bit conditional move FPU instruction.
-    FpuCMov32 {
-        rd: Writable<Reg>,
-        cond: Cond,
-        rm: Reg,
-    },
-    /// A 64-bit conditional move FPU instruction.
-    FpuCMov64 {
-        rd: Writable<Reg>,
-        cond: Cond,
-        rm: Reg,
-    },
-
-    /// A 64-bit move instruction from GPR to FPR.
-    MovToFpr {
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-    /// A 64-bit move instruction from FPR to GPR.
-    MovFromFpr {
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-
-    /// 1-op FPU instruction.
-    FpuRR {
-        fpu_op: FPUOp1,
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-
-    /// 2-op FPU instruction.
-    FpuRRR {
-        fpu_op: FPUOp2,
-        rd: Writable<Reg>,
-        rm: Reg,
-    },
-
-    /// 3-op FPU instruction.
-    FpuRRRR {
-        fpu_op: FPUOp3,
-        rd: Writable<Reg>,
-        rn: Reg,
-        rm: Reg,
-    },
-
-    /// FPU copy sign instruction.
-    FpuCopysign {
-        rd: Writable<Reg>,
-        rn: Reg,
-        rm: Reg,
-    },
-
-    /// FPU comparison, single-precision (32 bit).
-    FpuCmp32 {
-        rn: Reg,
-        rm: Reg,
-    },
-
-    /// FPU comparison, double-precision (64 bit).
-    FpuCmp64 {
-        rn: Reg,
-        rm: Reg,
-    },
-
-    /// Floating-point load, single-precision (32 bit).
-    FpuLoad32 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// Floating-point store, single-precision (32 bit).
-    FpuStore32 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// Floating-point load, double-precision (64 bit).
-    FpuLoad64 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// Floating-point store, double-precision (64 bit).
-    FpuStore64 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// Floating-point byte-reversed load, single-precision (32 bit).
-    FpuLoadRev32 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// Floating-point byte-reversed store, single-precision (32 bit).
-    FpuStoreRev32 {
-        rd: Reg,
-        mem: MemArg,
-    },
-    /// Floating-point byte-reversed load, double-precision (64 bit).
-    FpuLoadRev64 {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-    /// Floating-point byte-reversed store, double-precision (64 bit).
-    FpuStoreRev64 {
-        rd: Reg,
-        mem: MemArg,
-    },
-
-    LoadFpuConst32 {
-        rd: Writable<Reg>,
-        const_data: u32,
-    },
-
-    LoadFpuConst64 {
-        rd: Writable<Reg>,
-        const_data: u64,
-    },
-
-    /// Conversion: FP -> integer.
-    FpuToInt {
-        op: FpuToIntOp,
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-
-    /// Conversion: integer -> FP.
-    IntToFpu {
-        op: IntToFpuOp,
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-
-    /// Round to integer.
-    FpuRound {
-        op: FpuRoundMode,
-        rd: Writable<Reg>,
-        rn: Reg,
-    },
-
-    /// 2-op FPU instruction implemented as vector instruction with the W bit.
-    FpuVecRRR {
-        fpu_op: FPUOp2,
-        rd: Writable<Reg>,
-        rn: Reg,
-        rm: Reg,
-    },
-
-    /// A machine call instruction.
-    Call {
-        link: Writable<Reg>,
-        info: Box<CallInfo>,
-    },
-    /// A machine indirect-call instruction.
-    CallInd {
-        link: Writable<Reg>,
-        info: Box<CallIndInfo>,
-    },
-
-    // ---- branches (exactly one must appear at end of BB) ----
-    /// A machine return instruction.
-    Ret {
-        link: Reg,
-    },
-
-    /// A placeholder instruction, generating no code, meaning that a function epilogue must be
-    /// inserted there.
-    EpiloguePlaceholder,
-
-    /// An unconditional branch.
-    Jump {
-        dest: BranchTarget,
-    },
-
-    /// A conditional branch. Contains two targets; at emission time, both are emitted, but
-    /// the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the
-    /// choice of taken/not_taken (inverting the branch polarity as needed) based on the
-    /// fallthrough at the time of lowering.
-    CondBr {
-        taken: BranchTarget,
-        not_taken: BranchTarget,
-        cond: Cond,
-    },
-
-    /// A conditional trap: execute a `Trap` if the condition is true. This is
-    /// one VCode instruction because it uses embedded control flow; it is
-    /// logically a single-in, single-out region, but needs to appear as one
-    /// unit to the register allocator.
-    ///
-    /// The `Cond` gives the conditional-branch condition that will
-    /// *execute* the embedded `Trap`. (In the emitted code, we use the inverse
-    /// of this condition in a branch that skips the trap instruction.)
-    TrapIf {
-        cond: Cond,
-        trap_code: TrapCode,
-    },
-
-    /// A one-way conditional branch, invisible to the CFG processing; used *only* as part of
-    /// straight-line sequences in code to be emitted.
-    ///
-    /// In more detail:
-    /// - This branch is lowered to a branch at the machine-code level, but does not end a basic
-    ///   block, and does not create edges in the CFG seen by regalloc.
-    /// - Thus, it is *only* valid to use as part of a single-in, single-out sequence that is
-    ///   lowered from a single CLIF instruction. For example, certain arithmetic operations may
-    ///   use these branches to handle certain conditions, such as overflows, traps, etc.
-    ///
-    /// See, e.g., the lowering of `trapif` (conditional trap) for an example.
-    OneWayCondBr {
-        target: BranchTarget,
-        cond: Cond,
-    },
-
-    /// An indirect branch through a register, augmented with set of all
-    /// possible successors.
-    IndirectBr {
-        rn: Reg,
-        targets: Vec<MachLabel>,
-    },
-
-    /// A "debugtrap" instruction, used for e.g. traps and debug breakpoints.
-    Debugtrap,
-
-    /// An instruction guaranteed to always be undefined and to trigger an illegal instruction at
-    /// runtime.
-    Trap {
-        trap_code: TrapCode,
-    },
-
-    /// Jump-table sequence, as one compound instruction (see note in lower.rs
-    /// for rationale).
-    JTSequence {
-        info: Box<JTSequenceInfo>,
-        ridx: Reg,
-        rtmp1: Writable<Reg>,
-        rtmp2: Writable<Reg>,
-    },
-
-    /// Load an inline symbol reference with RelocDistance::Far.
-    LoadExtNameFar {
-        rd: Writable<Reg>,
-        name: Box<ExternalName>,
-        offset: i64,
-    },
-
-    /// Load address referenced by `mem` into `rd`.
-    LoadAddr {
-        rd: Writable<Reg>,
-        mem: MemArg,
-    },
-
-    /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This
-    /// controls how MemArg::NominalSPOffset args are lowered.
-    VirtualSPOffsetAdj {
-        offset: i64,
-    },
-
-    /// A definition of a value label.
-    ValueLabelMarker {
-        reg: Reg,
-        label: ValueLabel,
-    },
-
-    /// An unwind pseudoinstruction describing the state of the
-    /// machine at this program point.
-    Unwind {
-        inst: UnwindInst,
-    },
-}
-
 #[test]
 fn inst_size_test() {
     // This test will help with unintentionally growing the size
@@ -984,6 +74,18 @@ fn inst_size_test() {
     assert_eq!(32, std::mem::size_of::<Inst>());
 }
 
+/// Supported instruction sets
+#[allow(non_camel_case_types)]
+#[derive(Debug)]
+pub(crate) enum InstructionSet {
+    /// Baseline ISA for cranelift is z14.
+    Base,
+    /// Miscellaneous-Instruction-Extensions Facility 2 (z15)
+    MIE2,
+    /// Vector-Enhancements Facility 2 (z15)
+    VXRS_EXT2,
+}
+
 impl Inst {
     /// Retrieve the ISA feature set in which the instruction is available.
     fn available_in_isa(&self) -> InstructionSet {
@@ -1103,8 +205,16 @@ impl Inst {
             | Inst::Unwind { .. } => InstructionSet::Base,
 
             // These depend on the opcode
-            Inst::AluRRR { alu_op, .. } => alu_op.available_from(),
-            Inst::UnaryRR { op, .. } => op.available_from(),
+            Inst::AluRRR { alu_op, .. } => match alu_op {
+                ALUOp::AndNot32 | ALUOp::AndNot64 => InstructionSet::MIE2,
+                ALUOp::OrrNot32 | ALUOp::OrrNot64 => InstructionSet::MIE2,
+                ALUOp::XorNot32 | ALUOp::XorNot64 => InstructionSet::MIE2,
+                _ => InstructionSet::Base,
+            },
+            Inst::UnaryRR { op, .. } => match op {
+                UnaryOp::PopcntReg => InstructionSet::MIE2,
+                _ => InstructionSet::Base,
+            },
 
             // These are all part of VXRS_EXT2
             Inst::FpuLoadRev32 { .. }
@@ -1601,29 +711,8 @@ fn s390x_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
 //=============================================================================
 // Instructions: map_regs
 
-fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
-    fn map_use<RUM: RegUsageMapper>(m: &RUM, r: &mut Reg) {
-        if r.is_virtual() {
-            let new = m.get_use(r.to_virtual_reg()).unwrap().to_reg();
-            *r = new;
-        }
-    }
-
-    fn map_def<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
-        if r.to_reg().is_virtual() {
-            let new = m.get_def(r.to_reg().to_virtual_reg()).unwrap().to_reg();
-            *r = Writable::from_reg(new);
-        }
-    }
-
-    fn map_mod<RUM: RegUsageMapper>(m: &RUM, r: &mut Writable<Reg>) {
-        if r.to_reg().is_virtual() {
-            let new = m.get_mod(r.to_reg().to_virtual_reg()).unwrap().to_reg();
-            *r = Writable::from_reg(new);
-        }
-    }
-
-    fn map_mem<RUM: RegUsageMapper>(m: &RUM, mem: &mut MemArg) {
+pub fn s390x_map_regs<RM: RegMapper>(inst: &mut Inst, mapper: &RM) {
+    fn map_mem<RM: RegMapper>(m: &RM, mem: &mut MemArg) {
         match mem {
             &mut MemArg::BXD12 {
                 ref mut base,
@@ -1636,14 +725,14 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
                 ..
             } => {
                 if *base != zero_reg() {
-                    map_use(m, base);
+                    m.map_use(base);
                 }
                 if *index != zero_reg() {
-                    map_use(m, index);
+                    m.map_use(index);
                 }
             }
             &mut MemArg::Label { .. } | &mut MemArg::Symbol { .. } => {}
-            &mut MemArg::RegOffset { ref mut reg, .. } => map_use(m, reg),
+            &mut MemArg::RegOffset { ref mut reg, .. } => m.map_use(reg),
             &mut MemArg::InitialSPOffset { .. } | &mut MemArg::NominalSPOffset { .. } => {}
         };
     }
@@ -1655,24 +744,24 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rm,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::AluRRSImm16 {
             ref mut rd,
             ref mut rn,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::AluRX {
             ref mut rd,
             ref mut mem,
             ..
         } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::AluRR {
@@ -1680,49 +769,49 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rm,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_mod(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::AluRSImm16 { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::AluRSImm32 { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::AluRUImm32 { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::AluRUImm16Shifted { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::AluRUImm32Shifted { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::SMulWide {
             ref mut rn,
             ref mut rm,
             ..
         } => {
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::UMulWide { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::SDivMod32 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::SDivMod64 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::UDivMod32 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::UDivMod64 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::Flogr { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::ShiftRR {
             ref mut rd,
@@ -1730,10 +819,10 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut shift_reg,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
             if *shift_reg != zero_reg() {
-                map_use(mapper, shift_reg);
+                mapper.map_use(shift_reg);
             }
         }
         &mut Inst::UnaryRR {
@@ -1741,47 +830,47 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rn,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::CmpRR {
             ref mut rn,
             ref mut rm,
             ..
         } => {
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::CmpRX {
             ref mut rn,
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
             map_mem(mapper, mem);
         }
         &mut Inst::CmpRSImm16 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::CmpRSImm32 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::CmpRUImm32 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::CmpTrapRR {
             ref mut rn,
             ref mut rm,
             ..
         } => {
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::CmpTrapRSImm16 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::CmpTrapRUImm16 { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
 
         &mut Inst::AtomicRmw {
@@ -1790,8 +879,8 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
             map_mem(mapper, mem);
         }
         &mut Inst::AtomicCas32 {
@@ -1800,8 +889,8 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_mod(rd);
+            mapper.map_use(rn);
             map_mem(mapper, mem);
         }
         &mut Inst::AtomicCas64 {
@@ -1810,8 +899,8 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_mod(rd);
+            mapper.map_use(rn);
             map_mem(mapper, mem);
         }
         &mut Inst::Fence => {}
@@ -1821,7 +910,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load32ZExt8 {
@@ -1829,7 +918,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load32SExt8 {
@@ -1837,7 +926,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load32ZExt16 {
@@ -1845,7 +934,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load32SExt16 {
@@ -1853,7 +942,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load64 {
@@ -1861,7 +950,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load64ZExt8 {
@@ -1869,7 +958,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load64SExt8 {
@@ -1877,7 +966,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load64ZExt16 {
@@ -1885,7 +974,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load64SExt16 {
@@ -1893,7 +982,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load64ZExt32 {
@@ -1901,7 +990,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Load64SExt32 {
@@ -1909,7 +998,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::LoadRev16 {
@@ -1917,7 +1006,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::LoadRev32 {
@@ -1925,7 +1014,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::LoadRev64 {
@@ -1933,7 +1022,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
 
@@ -1942,7 +1031,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Store16 {
@@ -1950,7 +1039,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Store32 {
@@ -1958,7 +1047,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::Store64 {
@@ -1966,7 +1055,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::StoreImm8 { ref mut mem, .. } => {
@@ -1986,7 +1075,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::StoreRev32 {
@@ -1994,7 +1083,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::StoreRev64 {
@@ -2002,7 +1091,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::LoadMultiple64 { .. } => {
@@ -2022,121 +1111,121 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rd,
             ref mut rm,
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_def(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::Mov32 {
             ref mut rd,
             ref mut rm,
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_def(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::Mov32Imm { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::Mov32SImm16 { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::Mov64SImm16 { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::Mov64SImm32 { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::Mov64UImm16Shifted { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::Mov64UImm32Shifted { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::Insert64UImm16Shifted { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::Insert64UImm32Shifted { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::CMov64 {
             ref mut rd,
             ref mut rm,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_mod(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::CMov32 {
             ref mut rd,
             ref mut rm,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_mod(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::CMov32SImm16 { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::CMov64SImm16 { ref mut rd, .. } => {
-            map_mod(mapper, rd);
+            mapper.map_mod(rd);
         }
         &mut Inst::FpuMove32 {
             ref mut rd,
             ref mut rn,
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::FpuMove64 {
             ref mut rd,
             ref mut rn,
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::FpuCMov64 {
             ref mut rd,
             ref mut rm,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_mod(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::FpuCMov32 {
             ref mut rd,
             ref mut rm,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_mod(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::MovToFpr {
             ref mut rd,
             ref mut rn,
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::MovFromFpr {
             ref mut rd,
             ref mut rn,
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::FpuRR {
             ref mut rd,
             ref mut rn,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::FpuRRR {
             ref mut rd,
             ref mut rm,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rm);
+            mapper.map_mod(rd);
+            mapper.map_use(rm);
         }
         &mut Inst::FpuRRRR {
             ref mut rd,
@@ -2144,9 +1233,9 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rm,
             ..
         } => {
-            map_mod(mapper, rd);
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_mod(rd);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::FpuCopysign {
             ref mut rd,
@@ -2154,30 +1243,30 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rm,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::FpuCmp32 {
             ref mut rn,
             ref mut rm,
         } => {
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::FpuCmp64 {
             ref mut rn,
             ref mut rm,
         } => {
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::FpuLoad32 {
             ref mut rd,
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::FpuLoad64 {
@@ -2185,7 +1274,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::FpuStore32 {
@@ -2193,7 +1282,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::FpuStore64 {
@@ -2201,7 +1290,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::FpuLoadRev32 {
@@ -2209,7 +1298,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::FpuLoadRev64 {
@@ -2217,7 +1306,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::FpuStoreRev32 {
@@ -2225,7 +1314,7 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::FpuStoreRev64 {
@@ -2233,38 +1322,38 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut mem,
             ..
         } => {
-            map_use(mapper, rd);
+            mapper.map_use(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::LoadFpuConst32 { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::LoadFpuConst64 { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::FpuToInt {
             ref mut rd,
             ref mut rn,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::IntToFpu {
             ref mut rd,
             ref mut rn,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::FpuRound {
             ref mut rd,
             ref mut rn,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::FpuVecRRR {
             ref mut rd,
@@ -2272,28 +1361,28 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rm,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
-            map_use(mapper, rm);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
+            mapper.map_use(rm);
         }
         &mut Inst::Extend {
             ref mut rd,
             ref mut rn,
             ..
         } => {
-            map_def(mapper, rd);
-            map_use(mapper, rn);
+            mapper.map_def(rd);
+            mapper.map_use(rn);
         }
         &mut Inst::Call {
             ref mut link,
             ref mut info,
         } => {
-            map_def(mapper, link);
+            mapper.map_def(link);
             for r in info.uses.iter_mut() {
-                map_use(mapper, r);
+                mapper.map_use(r);
             }
             for r in info.defs.iter_mut() {
-                map_def(mapper, r);
+                mapper.map_def(r);
             }
         }
         &mut Inst::CallInd {
@@ -2301,20 +1390,20 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut info,
             ..
         } => {
-            map_def(mapper, link);
+            mapper.map_def(link);
             for r in info.uses.iter_mut() {
-                map_use(mapper, r);
+                mapper.map_use(r);
             }
             for r in info.defs.iter_mut() {
-                map_def(mapper, r);
+                mapper.map_def(r);
             }
-            map_use(mapper, &mut info.rn);
+            mapper.map_use(&mut info.rn);
         }
         &mut Inst::Ret { .. } => {}
         &mut Inst::EpiloguePlaceholder => {}
         &mut Inst::Jump { .. } => {}
         &mut Inst::IndirectBr { ref mut rn, .. } => {
-            map_use(mapper, rn);
+            mapper.map_use(rn);
         }
         &mut Inst::CondBr { .. } | &mut Inst::OneWayCondBr { .. } => {}
         &mut Inst::Debugtrap | &mut Inst::Trap { .. } | &mut Inst::TrapIf { .. } => {}
@@ -2325,23 +1414,23 @@ fn s390x_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
             ref mut rtmp2,
             ..
         } => {
-            map_use(mapper, ridx);
-            map_def(mapper, rtmp1);
-            map_def(mapper, rtmp2);
+            mapper.map_use(ridx);
+            mapper.map_def(rtmp1);
+            mapper.map_def(rtmp2);
         }
         &mut Inst::LoadExtNameFar { ref mut rd, .. } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
         }
         &mut Inst::LoadAddr {
             ref mut rd,
             ref mut mem,
         } => {
-            map_def(mapper, rd);
+            mapper.map_def(rd);
             map_mem(mapper, mem);
         }
         &mut Inst::VirtualSPOffsetAdj { .. } => {}
         &mut Inst::ValueLabelMarker { ref mut reg, .. } => {
-            map_use(mapper, reg);
+            mapper.map_use(reg);
         }
         &mut Inst::Unwind { .. } => {}
     }
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
new file mode 100644
index 000000000000..ae464c5cc91b
--- /dev/null
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -0,0 +1,1780 @@
+;; s390x instruction selection and CLIF-to-MachInst lowering.
+
+;; The main lowering constructor term: takes a clif `Inst` and returns the
+;; register(s) within which the lowered instruction's result values live.
+(decl lower (Inst) ValueRegs)
+
+
+;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty (iconst (u64_from_imm64 n))))
+      (value_reg (imm ty n)))
+
+
+;;;; Rules for `bconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty (bconst $false)))
+      (value_reg (imm ty 0)))
+(rule (lower (has_type ty (bconst $true)))
+      (value_reg (imm ty 1)))
+
+
+;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (f32const (u64_from_ieee32 x)))
+      (value_reg (imm $F32 x)))
+
+
+;;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (f64const (u64_from_ieee64 x)))
+      (value_reg (imm $F64 x)))
+
+
+;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty (null)))
+      (value_reg (imm ty 0)))
+
+
+;;;; Rules for `nop` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (nop))
+      (value_reg (invalid_reg)))
+
+
+;;;; Rules for `copy` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (copy x))
+      (value_reg (put_in_reg x)))
+
+
+;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add two registers.
+(rule (lower (has_type (fits_in_64 ty) (iadd x y)))
+      (value_reg (add_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; Add a register and a sign-extended register.
+(rule (lower (has_type (fits_in_64 ty) (iadd x (sext32_value y))))
+      (value_reg (add_reg_sext32 ty (put_in_reg x) (put_in_reg y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd (sext32_value x) y)))
+      (value_reg (add_reg_sext32 ty (put_in_reg y) (put_in_reg x))))
+
+;; Add a register and an immediate.
+(rule (lower (has_type (fits_in_64 ty) (iadd x (i16_from_value y))))
+      (value_reg (add_simm16 ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (iadd (i16_from_value x) y)))
+      (value_reg (add_simm16 ty (put_in_reg y) x)))
+(rule (lower (has_type (fits_in_64 ty) (iadd x (i32_from_value y))))
+      (value_reg (add_simm32 ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (iadd (i32_from_value x) y)))
+      (value_reg (add_simm32 ty (put_in_reg y) x)))
+
+;; Add a register and memory (32/64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (iadd x (sinkable_load_32_64 y))))
+      (value_reg (add_mem ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd (sinkable_load_32_64 x) y)))
+      (value_reg (add_mem ty (put_in_reg y) (sink_load x))))
+
+;; Add a register and memory (16-bit types).
+(rule (lower (has_type (fits_in_64 ty) (iadd x (sinkable_load_16 y))))
+      (value_reg (add_mem_sext16 ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd (sinkable_load_16 x) y)))
+      (value_reg (add_mem_sext16 ty (put_in_reg y) (sink_load x))))
+
+;; Add a register and sign-extended memory.
+(rule (lower (has_type (fits_in_64 ty) (iadd x (sinkable_sload16 y))))
+      (value_reg (add_mem_sext16 ty (put_in_reg x) (sink_sload16 y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd (sinkable_sload16 x) y)))
+      (value_reg (add_mem_sext16 ty (put_in_reg y) (sink_sload16 x))))
+(rule (lower (has_type (fits_in_64 ty) (iadd x (sinkable_sload32 y))))
+      (value_reg (add_mem_sext32 ty (put_in_reg x) (sink_sload32 y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd (sinkable_sload32 x) y)))
+      (value_reg (add_mem_sext32 ty (put_in_reg y) (sink_sload32 x))))
+
+
+;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Sub two registers.
+(rule (lower (has_type (fits_in_64 ty) (isub x y)))
+      (value_reg (sub_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; Sub a register and a sign-extended register.
+(rule (lower (has_type (fits_in_64 ty) (isub x (sext32_value y))))
+      (value_reg (sub_reg_sext32 ty (put_in_reg x) (put_in_reg y))))
+
+;; Sub a register and an immediate (using add of the negated value).
+(rule (lower (has_type (fits_in_64 ty) (isub x (i16_from_negated_value y))))
+      (value_reg (add_simm16 ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (isub x (i32_from_negated_value y))))
+      (value_reg (add_simm32 ty (put_in_reg x) y)))
+
+;; Sub a register and memory (32/64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (isub x (sinkable_load_32_64 y))))
+      (value_reg (sub_mem ty (put_in_reg x) (sink_load y))))
+
+;; Sub a register and memory (16-bit types).
+(rule (lower (has_type (fits_in_64 ty) (isub x (sinkable_load_16 y))))
+      (value_reg (sub_mem_sext16 ty (put_in_reg x) (sink_load y))))
+
+;; Sub a register and sign-extended memory.
+(rule (lower (has_type (fits_in_64 ty) (isub x (sinkable_sload16 y))))
+      (value_reg (sub_mem_sext16 ty (put_in_reg x) (sink_sload16 y))))
+(rule (lower (has_type (fits_in_64 ty) (isub x (sinkable_sload32 y))))
+      (value_reg (sub_mem_sext32 ty (put_in_reg x) (sink_sload32 y))))
+
+
+;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Absolute value of a register.
+;; For types smaller than 32-bit, the input value must be sign-extended.
+(rule (lower (has_type (fits_in_64 ty) (iabs x)))
+      (value_reg (abs_reg (ty_ext32 ty) (put_in_reg_sext32 x))))
+
+;; Absolute value of a sign-extended register.
+(rule (lower (has_type (fits_in_64 ty) (iabs (sext32_value x))))
+      (value_reg (abs_reg_sext32 ty (put_in_reg x))))
+
+
+;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; N.B.: the second output of `iadd_ifcout` is meant to be the `iflags` value
+;; containing the carry result, but we do not support the `iflags` mechanism.
+;; However, the only actual use case is where `iadd_ifcout` feeds into `trapif`,
+;; which is implemented by explicitly matching on the flags producer.  So we can
+;; get away with just allocating a second temp so that the reg-renaming code
+;; does the right thing, for now.
+(decl value_regs_ifcout (Reg) ValueRegs)
+(rule (value_regs_ifcout reg)
+      (value_regs reg (writable_reg_to_reg (temp_writable_reg $I64))))
+
+;; Add two registers.
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x y)))
+      (value_regs_ifcout (add_logical_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; Add a register and a zero-extended register.
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (zext32_value y))))
+      (value_regs_ifcout (add_logical_reg_zext32 ty (put_in_reg x) (put_in_reg y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (zext32_value x) y)))
+      (value_regs_ifcout (add_logical_reg_zext32 ty (put_in_reg y) (put_in_reg x))))
+
+;; Add a register and an immediate.
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (u32_from_value y))))
+      (value_regs_ifcout (add_logical_zimm32 ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (u32_from_value x) y)))
+      (value_regs_ifcout (add_logical_zimm32 ty (put_in_reg y) x)))
+
+;; Add a register and memory (32/64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (sinkable_load_32_64 y))))
+      (value_regs_ifcout (add_logical_mem ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (sinkable_load_32_64 x) y)))
+      (value_regs_ifcout (add_logical_mem ty (put_in_reg y) (sink_load x))))
+
+;; Add a register and zero-extended memory.
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout x (sinkable_uload32 y))))
+      (value_regs_ifcout (add_logical_mem_zext32 ty (put_in_reg x) (sink_uload32 y))))
+(rule (lower (has_type (fits_in_64 ty) (iadd_ifcout (sinkable_uload32 x) y)))
+      (value_regs_ifcout (add_logical_mem_zext32 ty (put_in_reg y) (sink_uload32 x))))
+
+
+;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Negate a register.
+(rule (lower (has_type (fits_in_64 ty) (ineg x)))
+      (value_reg (neg_reg ty (put_in_reg x))))
+
+;; Negate a sign-extended register.
+(rule (lower (has_type (fits_in_64 ty) (ineg (sext32_value x))))
+      (value_reg (neg_reg_sext32 ty (put_in_reg x))))
+
+
+;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Multiply two registers.
+(rule (lower (has_type (fits_in_64 ty) (imul x y)))
+      (value_reg (mul_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; Multiply a register and a sign-extended register.
+(rule (lower (has_type (fits_in_64 ty) (imul x (sext32_value y))))
+      (value_reg (mul_reg_sext32 ty (put_in_reg x) (put_in_reg y))))
+(rule (lower (has_type (fits_in_64 ty) (imul (sext32_value x) y)))
+      (value_reg (mul_reg_sext32 ty (put_in_reg y) (put_in_reg x))))
+
+;; Multiply a register and an immediate.
+(rule (lower (has_type (fits_in_64 ty) (imul x (i16_from_value y))))
+      (value_reg (mul_simm16 ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (imul (i16_from_value x) y)))
+      (value_reg (mul_simm16 ty (put_in_reg y) x)))
+(rule (lower (has_type (fits_in_64 ty) (imul x (i32_from_value y))))
+      (value_reg (mul_simm32 ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (imul (i32_from_value x) y)))
+      (value_reg (mul_simm32 ty (put_in_reg y) x)))
+
+;; Multiply a register and memory (32/64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (imul x (sinkable_load_32_64 y))))
+      (value_reg (mul_mem ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (imul (sinkable_load_32_64 x) y)))
+      (value_reg (mul_mem ty (put_in_reg y) (sink_load x))))
+
+;; Multiply a register and memory (16-bit types).
+(rule (lower (has_type (fits_in_64 ty) (imul x (sinkable_load_16 y))))
+      (value_reg (mul_mem_sext16 ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (imul (sinkable_load_16 x) y)))
+      (value_reg (mul_mem_sext16 ty (put_in_reg y) (sink_load x))))
+
+;; Multiply a register and sign-extended memory.
+(rule (lower (has_type (fits_in_64 ty) (imul x (sinkable_sload16 y))))
+      (value_reg (mul_mem_sext16 ty (put_in_reg x) (sink_sload16 y))))
+(rule (lower (has_type (fits_in_64 ty) (imul (sinkable_sload16 x) y)))
+      (value_reg (mul_mem_sext16 ty (put_in_reg y) (sink_sload16 x))))
+(rule (lower (has_type (fits_in_64 ty) (imul x (sinkable_sload32 y))))
+      (value_reg (mul_mem_sext32 ty (put_in_reg x) (sink_sload32 y))))
+(rule (lower (has_type (fits_in_64 ty) (imul (sinkable_sload32 x) y)))
+      (value_reg (mul_mem_sext32 ty (put_in_reg y) (sink_sload32 x))))
+
+
+;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Multiply high part unsigned, 8-bit or 16-bit types.  (Uses 32-bit multiply.)
+(rule (lower (has_type (ty_8_or_16 ty) (umulhi x y)))
+      (let ((ext_reg_x Reg (put_in_reg_zext32 x))
+            (ext_reg_y Reg (put_in_reg_zext32 y))
+            (ext_mul Reg (mul_reg $I32 ext_reg_x ext_reg_y)))
+        (value_reg (lshr_imm $I32 ext_mul (ty_bits ty)))))
+
+;; Multiply high part unsigned, 32-bit types.  (Uses 64-bit multiply.)
+(rule (lower (has_type $I32 (umulhi x y)))
+      (let ((ext_reg_x Reg (put_in_reg_zext64 x))
+            (ext_reg_y Reg (put_in_reg_zext64 y))
+            (ext_mul Reg (mul_reg $I64 ext_reg_x ext_reg_y)))
+        (value_reg (lshr_imm $I64 ext_mul 32))))
+
+;; Multiply high part unsigned, 64-bit types.  (Uses umul_wide.)
+(rule (lower (has_type $I64 (umulhi x y)))
+      (let ((pair RegPair (umul_wide (put_in_reg x) (put_in_reg y))))
+        (value_reg (copy_reg $I64 (regpair_hi pair)))))
+
+
+;;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Multiply high part signed, 8-bit or 16-bit types.  (Uses 32-bit multiply.)
+(rule (lower (has_type (ty_8_or_16 ty) (smulhi x y)))
+      (let ((ext_reg_x Reg (put_in_reg_sext32 x))
+            (ext_reg_y Reg (put_in_reg_sext32 y))
+            (ext_mul Reg (mul_reg $I32 ext_reg_x ext_reg_y)))
+        (value_reg (ashr_imm $I32 ext_mul (ty_bits ty)))))
+
+;; Multiply high part signed, 32-bit types.  (Uses 64-bit multiply.)
+(rule (lower (has_type $I32 (smulhi x y)))
+      (let ((ext_reg_x Reg (put_in_reg_sext64 x))
+            (ext_reg_y Reg (put_in_reg_sext64 y))
+            (ext_mul Reg (mul_reg $I64 ext_reg_x ext_reg_y)))
+        (value_reg (ashr_imm $I64 ext_mul 32))))
+
+;; Multiply high part signed, 64-bit types.  (Uses smul_wide.)
+(rule (lower (has_type $I64 (smulhi x y)))
+      (let ((pair RegPair (smul_wide (put_in_reg x) (put_in_reg y))))
+        (value_reg (copy_reg $I64 (regpair_hi pair)))))
+
+
+;;;; Rules for `udiv` and `urem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Divide two registers.  The architecture provides combined udiv / urem
+;; instructions with the following combination of data types:
+;;
+;;  - 64-bit dividend (split across a 2x32-bit register pair),
+;;    32-bit divisor (in a single input register)
+;;    32-bit quotient & remainder (in a 2x32-bit register pair)
+;;
+;;  - 128-bit dividend (split across a 2x64-bit register pair),
+;;    64-bit divisor (in a single input register)
+;;    64-bit quotient & remainder (in a 2x64-bit register pair)
+;;
+;; We use the first variant for 32-bit and smaller input types,
+;; and the second variant for 64-bit input types.
+
+;; Implement `udiv`.
+(rule (lower (has_type (fits_in_64 ty) (udiv x y)))
+      (let (;; Look at the divisor to determine whether we need to generate
+            ;; an explicit division-by zero check.
+            (DZcheck bool (zero_divisor_check_needed y))
+            ;; Load up the dividend, by loading the input (possibly zero-
+            ;; extended) input into the low half of the register pair,
+            ;; and setting the high half to zero.
+            (ext_x RegPair (put_in_regpair_lo_zext32 x
+                             (imm_regpair_hi (ty_ext32 ty) 0 (uninitialized_regpair))))
+            ;; Load up the divisor, zero-extended if necessary.
+            (ext_y Reg (put_in_reg_zext32 y))
+            (ext_ty Type (ty_ext32 ty))
+            ;; Now actually perform the division-by zero check if necessary.
+            ;; This cannot be done earlier than here, because the check
+            ;; requires an already extended divisor value.
+            (_ Reg (maybe_trap_if_zero_divisor DZcheck ext_ty ext_y))
+            ;; Emit the actual divide instruction.
+            (pair RegPair (udivmod ext_ty ext_x ext_y)))
+        ;; The quotient can be found in the low half of the result.
+        (value_reg (copy_reg ty (regpair_lo pair)))))
+
+;; Implement `urem`.  Same as `udiv`, but finds the remainder in
+;; the high half of the result register pair instead.
+(rule (lower (has_type (fits_in_64 ty) (urem x y)))
+      (let ((DZcheck bool (zero_divisor_check_needed y))
+            (ext_x RegPair (put_in_regpair_lo_zext32 x
+                             (imm_regpair_hi ty 0 (uninitialized_regpair))))
+            (ext_y Reg (put_in_reg_zext32 y))
+            (ext_ty Type (ty_ext32 ty))
+            (_ Reg (maybe_trap_if_zero_divisor DZcheck ext_ty ext_y))
+            (pair RegPair (udivmod ext_ty ext_x ext_y)))
+        (value_reg (copy_reg ty (regpair_hi pair)))))
+
+;; Determine whether we need to perform a divide-by-zero-check.
+;;
+;; If the `avoid_div_traps` flag is false, we never need to perform
+;; that check; we can rely on the divide instruction itself to trap.
+;;
+;; If the `avoid_div_traps` flag is true, we perform the check explicitly.
+;; This still can be omittted if the divisor is a non-zero immediate.
+(decl zero_divisor_check_needed (Value) bool)
+(rule (zero_divisor_check_needed (i64_from_value (i64_nonzero _))) $false)
+(rule (zero_divisor_check_needed (value_type (allow_div_traps))) $false)
+(rule (zero_divisor_check_needed _) $true)
+
+;; Perform the divide-by-zero check if required.
+;; This is simply a compare-and-trap of the (extended) divisor against 0.
+(decl maybe_trap_if_zero_divisor (bool Type Reg) Reg)
+(rule (maybe_trap_if_zero_divisor $false _ _) (invalid_reg))
+(rule (maybe_trap_if_zero_divisor $true ext_ty reg)
+      (icmps_simm16_and_trap ext_ty reg 0
+                             (intcc_as_cond (IntCC.Equal))
+                             (trap_code_division_by_zero)))
+
+
+;;;; Rules for `sdiv` and `srem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Divide two registers.  The architecture provides combined sdiv / srem
+;; instructions with the following combination of data types:
+;;
+;;  - 64-bit dividend (in the low half of a 2x64-bit register pair),
+;;    32-bit divisor (in a single input register)
+;;    64-bit quotient & remainder (in a 2x64-bit register pair)
+;;
+;;  - 64-bit dividend (in the low half of a 2x64-bit register pair),
+;;    64-bit divisor (in a single input register)
+;;    64-bit quotient & remainder (in a 2x64-bit register pair)
+;;
+;; We use the first variant for 32-bit and smaller input types,
+;; and the second variant for 64-bit input types.
+
+;; Implement `sdiv`.
+(rule (lower (has_type (fits_in_64 ty) (sdiv x y)))
+      (let (;; Look at the divisor to determine whether we need to generate
+            ;; explicit division-by-zero and/or integer-overflow checks.
+            (DZcheck bool (zero_divisor_check_needed y))
+            (OFcheck bool (div_overflow_check_needed y))
+            ;; Load up the dividend (sign-extended to 64-bit) into the low
+            ;; half of a register pair (the high half remains uninitialized).
+            (ext_x RegPair (put_in_regpair_lo_sext64 x (uninitialized_regpair)))
+            ;; Load up the divisor (sign-extended if necessary).
+            (ext_y Reg (put_in_reg_sext32 y))
+            (ext_ty Type (ty_ext32 ty))
+            ;; Perform division-by-zero check (same as for `udiv`).
+            (_1 Reg (maybe_trap_if_zero_divisor DZcheck ext_ty ext_y))
+            ;; Perform integer-overflow check if necessary.
+            (_2 Reg (maybe_trap_if_sdiv_overflow OFcheck ext_ty ty ext_x ext_y))
+            ;; Emit the actual divide instruction.
+            (pair RegPair (sdivmod ext_ty ext_x ext_y)))
+        ;; The quotient can be found in the low half of the result.
+        (value_reg (copy_reg ty (regpair_lo pair)))))
+
+;; Implement `srem`.  Same as `sdiv`, but finds the remainder in
+;; the high half of the result register pair instead.  Also, handle
+;; the integer overflow case differently, see below.
+(rule (lower (has_type (fits_in_64 ty) (srem x y)))
+      (let ((DZcheck bool (zero_divisor_check_needed y))
+            (OFcheck bool (div_overflow_check_needed y))
+            (ext_x RegPair (put_in_regpair_lo_sext64 x (uninitialized_regpair)))
+            (ext_y Reg (put_in_reg_sext32 y))
+            (ext_ty Type (ty_ext32 ty))
+            (_ Reg (maybe_trap_if_zero_divisor DZcheck ext_ty ext_y))
+            (checked_x RegPair (maybe_avoid_srem_overflow OFcheck ext_ty ext_x ext_y))
+            (pair RegPair (sdivmod ext_ty checked_x ext_y)))
+        (value_reg (copy_reg ty (regpair_hi pair)))))
+
+;; Determine whether we need to perform an integer-overflow check.
+;;
+;; We never rely on the divide instruction itself to trap; while that trap
+;; would indeed happen, we have no way of signalling two different trap
+;; conditions from the same instruction.  By explicity checking for the
+;; integer-overflow case ahead of time, any hardware trap in the divide
+;; instruction is guaranteed to indicate divison-by-zero.
+;;
+;; In addition, for types smaller than 64 bits we would have to perform
+;; the check explicitly anyway, since the instruction provides a 64-bit
+;; quotient and only traps if *that* overflows.
+;;
+;; However, the only case where integer overflow can occur is if the
+;; minimum (signed) integer value is divided by -1, so if the divisor
+;; is any immediate different from -1, the check can be omitted.
+(decl div_overflow_check_needed (Value) bool)
+(rule (div_overflow_check_needed (i64_from_value (i64_not_neg1 _))) $false)
+(rule (div_overflow_check_needed _) $true)
+
+;; Perform the integer-overflow check if necessary.   This implements:
+;;
+;;    if divisor == INT_MIN && dividend == -1 { trap }
+;;
+;; but to avoid introducing control flow, it is actually done as:
+;;
+;;    if ((divisor ^ INT_MAX) & dividend) == -1 { trap }
+;;
+;; instead, using a single conditional trap instruction.
+(decl maybe_trap_if_sdiv_overflow (bool Type Type RegPair Reg) Reg)
+(rule (maybe_trap_if_sdiv_overflow $false ext_ty _ _ _) (invalid_reg))
+(rule (maybe_trap_if_sdiv_overflow $true ext_ty ty x y)
+      (let ((int_max Reg (imm ext_ty (int_max ty)))
+            (reg Reg (and_reg ext_ty (xor_reg ext_ty int_max
+                                              (regpair_lo x)) y)))
+        (icmps_simm16_and_trap ext_ty reg -1
+                               (intcc_as_cond (IntCC.Equal))
+                               (trap_code_integer_overflow))))
+(decl int_max (Type) u64)
+(rule (int_max $I8) 0x7f)
+(rule (int_max $I16) 0x7fff)
+(rule (int_max $I32) 0x7fffffff)
+(rule (int_max $I64) 0x7fffffffffffffff)
+
+;; When performing `srem`, we do not want to trap in the
+;; integer-overflow scenario, because it is only the quotient
+;; that overflows, not the remainder.
+;;
+;; For types smaller than 64 bits, we can simply let the
+;; instruction execute, since (as above) it will never trap.
+;;
+;; For 64-bit inputs, we check whether the divisor is -1, and
+;; if so simply replace the dividend by zero, which will give
+;; the correct result, since any value modulo -1 is zero.
+;;
+;; (We could in fact avoid executing the divide instruction
+;; at all in this case, but that would require introducing
+;; control flow.)
+(decl maybe_avoid_srem_overflow (bool Type RegPair Reg) RegPair)
+(rule (maybe_avoid_srem_overflow $false _ x _) x)
+(rule (maybe_avoid_srem_overflow $true $I32 x _) x)
+(rule (maybe_avoid_srem_overflow $true $I64 x y)
+      (cmov_imm_regpair_lo $I64 (icmps_simm16 $I64 y -1)
+                           (intcc_as_cond (IntCC.Equal)) 0 x))
+
+
+;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Shift left, shift amount in register.
+(rule (lower (has_type (fits_in_64 ty) (ishl x y)))
+      (let ((masked_amt Reg (mask_amt_reg ty (put_in_reg y))))
+        (value_reg (lshl_reg ty (put_in_reg x) masked_amt))))
+
+;; Shift left, immediate shift amount.
+(rule (lower (has_type (fits_in_64 ty) (ishl x (i64_from_value y))))
+      (let ((masked_amt u8 (mask_amt_imm ty y)))
+        (value_reg (lshl_imm ty (put_in_reg x) masked_amt))))
+
+
+;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Shift right logical, shift amount in register.
+;; For types smaller than 32-bit, the input value must be zero-extended.
+(rule (lower (has_type (fits_in_64 ty) (ushr x y)))
+      (let ((ext_reg Reg (put_in_reg_zext32 x))
+            (masked_amt Reg (mask_amt_reg ty (put_in_reg y))))
+        (value_reg (lshr_reg (ty_ext32 ty) ext_reg masked_amt))))
+
+;; Shift right logical, immediate shift amount.
+;; For types smaller than 32-bit, the input value must be zero-extended.
+(rule (lower (has_type (fits_in_64 ty) (ushr x (i64_from_value y))))
+      (let ((ext_reg Reg (put_in_reg_zext32 x))
+            (masked_amt u8 (mask_amt_imm ty y)))
+        (value_reg (lshr_imm (ty_ext32 ty) ext_reg masked_amt))))
+
+
+;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Shift right arithmetic, shift amount in register.
+;; For types smaller than 32-bit, the input value must be sign-extended.
+(rule (lower (has_type (fits_in_64 ty) (sshr x y)))
+      (let ((ext_reg Reg (put_in_reg_sext32 x))
+            (masked_amt Reg (mask_amt_reg ty (put_in_reg y))))
+        (value_reg (ashr_reg (ty_ext32 ty) ext_reg masked_amt))))
+
+;; Shift right arithmetic, immediate shift amount.
+;; For types smaller than 32-bit, the input value must be sign-extended.
+(rule (lower (has_type (fits_in_64 ty) (sshr x (i64_from_value y))))
+      (let ((ext_reg Reg (put_in_reg_sext32 x))
+            (masked_amt u8 (mask_amt_imm ty y)))
+        (value_reg (ashr_imm (ty_ext32 ty) ext_reg masked_amt))))
+
+
+;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Rotate left, shift amount in register.  32-bit or 64-bit types.
+(rule (lower (has_type (ty_32_or_64 ty) (rotl x y)))
+      (value_reg (rot_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; Rotate left arithmetic, immediate shift amount.  32-bit or 64-bit types.
+(rule (lower (has_type (ty_32_or_64 ty) (rotl x (i64_from_value y))))
+      (let ((masked_amt u8 (mask_amt_imm ty y)))
+        (value_reg (rot_imm ty (put_in_reg x) masked_amt))))
+
+;; Rotate left, shift amount in register.  8-bit or 16-bit types.
+;; Implemented via a pair of 32-bit shifts on the zero-extended input.
+(rule (lower (has_type (ty_8_or_16 ty) (rotl x y)))
+      (let ((ext_reg Reg (put_in_reg_zext32 x))
+            (ext_ty Type (ty_ext32 ty))
+            (pos_amt Reg (put_in_reg y))
+            (neg_amt Reg (neg_reg ty pos_amt))
+            (masked_pos_amt Reg (mask_amt_reg ty pos_amt))
+            (masked_neg_amt Reg (mask_amt_reg ty neg_amt)))
+        (value_reg (or_reg ty (lshl_reg ext_ty ext_reg masked_pos_amt)
+                              (lshr_reg ext_ty ext_reg masked_neg_amt)))))
+
+;; Rotate left, immediate shift amount.  8-bit or 16-bit types.
+;; Implemented via a pair of 32-bit shifts on the zero-extended input.
+(rule (lower (has_type (ty_8_or_16 ty) (rotl x (and (i64_from_value pos_amt)
+                                                    (i64_from_negated_value neg_amt)))))
+      (let ((ext_reg Reg (put_in_reg_zext32 x))
+            (ext_ty Type (ty_ext32 ty))
+            (masked_pos_amt u8 (mask_amt_imm ty pos_amt))
+            (masked_neg_amt u8 (mask_amt_imm ty neg_amt)))
+        (value_reg (or_reg ty (lshl_imm ext_ty ext_reg masked_pos_amt)
+                              (lshr_imm ext_ty ext_reg masked_neg_amt)))))
+
+
+;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Rotate right, shift amount in register.  32-bit or 64-bit types.
+;; Implemented as rotate left with negated rotate amount.
+(rule (lower (has_type (ty_32_or_64 ty) (rotr x y)))
+      (let ((negated_amt Reg (neg_reg ty (put_in_reg y))))
+        (value_reg (rot_reg ty (put_in_reg x) negated_amt))))
+
+;; Rotate right arithmetic, immediate shift amount.  32-bit or 64-bit types.
+;; Implemented as rotate left with negated rotate amount.
+(rule (lower (has_type (ty_32_or_64 ty) (rotr x (i64_from_negated_value y))))
+      (let ((negated_amt u8 (mask_amt_imm ty y)))
+        (value_reg (rot_imm ty (put_in_reg x) negated_amt))))
+
+;; Rotate right, shift amount in register.  8-bit or 16-bit types.
+;; Implemented as rotate left with negated rotate amount.
+(rule (lower (has_type (ty_8_or_16 ty) (rotr x y)))
+      (let ((ext_reg Reg (put_in_reg_zext32 x))
+            (ext_ty Type (ty_ext32 ty))
+            (pos_amt Reg (put_in_reg y))
+            (neg_amt Reg (neg_reg ty pos_amt))
+            (masked_pos_amt Reg (mask_amt_reg ty pos_amt))
+            (masked_neg_amt Reg (mask_amt_reg ty neg_amt)))
+        (value_reg (or_reg ty (lshl_reg ext_ty ext_reg masked_neg_amt)
+                              (lshr_reg ext_ty ext_reg masked_pos_amt)))))
+
+;; Rotate right, immediate shift amount.  8-bit or 16-bit types.
+;; Implemented as rotate left with negated rotate amount.
+(rule (lower (has_type (ty_8_or_16 ty) (rotr x (and (i64_from_value pos_amt)
+                                                    (i64_from_negated_value neg_amt)))))
+      (let ((ext_reg Reg (put_in_reg_zext32 x))
+            (ext_ty Type (ty_ext32 ty))
+            (masked_pos_amt u8 (mask_amt_imm ty pos_amt))
+            (masked_neg_amt u8 (mask_amt_imm ty neg_amt)))
+        (value_reg (or_reg ty (lshl_imm ext_ty ext_reg masked_neg_amt)
+                              (lshr_imm ext_ty ext_reg masked_pos_amt)))))
+
+;;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Always a no-op.
+(rule (lower (ireduce x))
+      (value_reg (put_in_reg x)))
+
+
+;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 16- or 32-bit target types.
+(rule (lower (has_type (gpr32_ty _ty) (uextend x)))
+      (value_reg (put_in_reg_zext32 x)))
+
+;; 64-bit target types.
+(rule (lower (has_type (gpr64_ty _ty) (uextend x)))
+      (value_reg (put_in_reg_zext64 x)))
+
+
+;;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 16- or 32-bit target types.
+(rule (lower (has_type (gpr32_ty _ty) (sextend x)))
+      (value_reg (put_in_reg_sext32 x)))
+
+;; 64-bit target types.
+(rule (lower (has_type (gpr64_ty _ty) (sextend x)))
+      (value_reg (put_in_reg_sext64 x)))
+
+
+;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; z15 version using a single instruction (NOR).
+(rule (lower (has_type (and (mie2_enabled) (fits_in_64 ty)) (bnot x)))
+      (let ((rx Reg (put_in_reg x)))
+        (value_reg (or_not_reg ty rx rx))))
+
+;; z14 version using XOR with -1.
+(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (bnot x)))
+      (value_reg (not_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; And two registers.
+(rule (lower (has_type (fits_in_64 ty) (band x y)))
+      (value_reg (and_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; And a register and an immediate.
+(rule (lower (has_type (fits_in_64 ty) (band x (uimm16shifted_from_inverted_value y))))
+      (value_reg (and_uimm16shifted ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (band (uimm16shifted_from_inverted_value x) y)))
+      (value_reg (and_uimm16shifted ty (put_in_reg y) x)))
+(rule (lower (has_type (fits_in_64 ty) (band x (uimm32shifted_from_inverted_value y))))
+      (value_reg (and_uimm32shifted ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (band (uimm32shifted_from_inverted_value x) y)))
+      (value_reg (and_uimm32shifted ty (put_in_reg y) x)))
+
+;; And a register and memory (32/64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (band x (sinkable_load_32_64 y))))
+      (value_reg (and_mem ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (band (sinkable_load_32_64 x) y)))
+      (value_reg (and_mem ty (put_in_reg y) (sink_load x))))
+
+
+;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Or two registers.
+(rule (lower (has_type (fits_in_64 ty) (bor x y)))
+      (value_reg (or_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; Or a register and an immediate.
+(rule (lower (has_type (fits_in_64 ty) (bor x (uimm16shifted_from_value y))))
+      (value_reg (or_uimm16shifted ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (bor (uimm16shifted_from_value x) y)))
+      (value_reg (or_uimm16shifted ty (put_in_reg y) x)))
+(rule (lower (has_type (fits_in_64 ty) (bor x (uimm32shifted_from_value y))))
+      (value_reg (or_uimm32shifted ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (bor (uimm32shifted_from_value x) y)))
+      (value_reg (or_uimm32shifted ty (put_in_reg y) x)))
+
+;; Or a register and memory (32/64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (bor x (sinkable_load_32_64 y))))
+      (value_reg (or_mem ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (bor (sinkable_load_32_64 x) y)))
+      (value_reg (or_mem ty (put_in_reg y) (sink_load x))))
+
+
+;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Xor two registers.
+(rule (lower (has_type (fits_in_64 ty) (bxor x y)))
+      (value_reg (xor_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; Xor a register and an immediate.
+(rule (lower (has_type (fits_in_64 ty) (bxor x (uimm32shifted_from_value y))))
+      (value_reg (xor_uimm32shifted ty (put_in_reg x) y)))
+(rule (lower (has_type (fits_in_64 ty) (bxor (uimm32shifted_from_value x) y)))
+      (value_reg (xor_uimm32shifted ty (put_in_reg y) x)))
+
+;; Xor a register and memory (32/64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (bxor x (sinkable_load_32_64 y))))
+      (value_reg (xor_mem ty (put_in_reg x) (sink_load y))))
+(rule (lower (has_type (fits_in_64 ty) (bxor (sinkable_load_32_64 x) y)))
+      (value_reg (xor_mem ty (put_in_reg y) (sink_load x))))
+
+
+;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; z15 version using a single instruction.
+(rule (lower (has_type (and (mie2_enabled) (fits_in_64 ty)) (band_not x y)))
+      (value_reg (and_not_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; z14 version using XOR with -1.
+(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (band_not x y)))
+      (value_reg (not_reg ty (and_reg ty (put_in_reg x) (put_in_reg y)))))
+
+
+;;;; Rules for `bor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; z15 version using a single instruction.
+(rule (lower (has_type (and (mie2_enabled) (fits_in_64 ty)) (bor_not x y)))
+      (value_reg (or_not_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; z14 version using XOR with -1.
+(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (bor_not x y)))
+      (value_reg (not_reg ty (or_reg ty (put_in_reg x) (put_in_reg y)))))
+
+
+;;;; Rules for `bxor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; z15 version using a single instruction.
+(rule (lower (has_type (and (mie2_enabled) (fits_in_64 ty)) (bxor_not x y)))
+      (value_reg (xor_not_reg ty (put_in_reg x) (put_in_reg y))))
+
+;; z14 version using XOR with -1.
+(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (bxor_not x y)))
+      (value_reg (not_reg ty (xor_reg ty (put_in_reg x) (put_in_reg y)))))
+
+
+;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; z15 version using a NAND instruction.
+(rule (lower (has_type (and (mie2_enabled) (fits_in_64 ty)) (bitselect x y z)))
+      (let ((rx Reg (put_in_reg x))
+            (if_true Reg (and_reg ty (put_in_reg y) rx))
+            (if_false Reg (and_not_reg ty (put_in_reg z) rx)))
+        (value_reg (or_reg ty if_false if_true))))
+
+;; z14 version using XOR with -1.
+(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (bitselect x y z)))
+      (let ((rx Reg (put_in_reg x))
+            (if_true Reg (and_reg ty (put_in_reg y) rx))
+            (if_false Reg (not_reg ty (and_reg ty (put_in_reg z) rx))))
+        (value_reg (or_reg ty if_false if_true))))
+
+
+;;;; Rules for `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Always a no-op.
+(rule (lower (breduce x))
+      (value_reg (put_in_reg x)))
+
+
+;;;; Rules for `bextend` and `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Use a common helper to type cast bools to either bool or integer types.
+(decl cast_bool (Type Value) Reg)
+(rule (lower (has_type ty (bextend x)))
+      (value_reg (cast_bool ty x)))
+(rule (lower (has_type ty (bmask x)))
+      (value_reg (cast_bool ty x)))
+
+;; If the target has the same or a smaller size than the source, it's a no-op.
+(rule (cast_bool $B1 x @ (value_type $B1)) (put_in_reg x))
+(rule (cast_bool $B1 x @ (value_type $B8)) (put_in_reg x))
+(rule (cast_bool $B8 x @ (value_type $B8)) (put_in_reg x))
+(rule (cast_bool $I8 x @ (value_type $B8)) (put_in_reg x))
+(rule (cast_bool (fits_in_16 _ty) x @ (value_type $B16)) (put_in_reg x))
+(rule (cast_bool (fits_in_32 _ty) x @ (value_type $B32)) (put_in_reg x))
+(rule (cast_bool (fits_in_64 _ty) x @ (value_type $B64)) (put_in_reg x))
+
+;; Single-bit values are sign-extended via a pair of shifts.
+(rule (cast_bool (gpr32_ty ty) x @ (value_type $B1))
+      (ashr_imm $I32 (lshl_imm $I32 (put_in_reg x) 31) 31))
+(rule (cast_bool (gpr64_ty ty) x @ (value_type $B1))
+      (ashr_imm $I64 (lshl_imm $I64 (put_in_reg x) 63) 63))
+
+;; Other values are just sign-extended normally.
+(rule (cast_bool (gpr32_ty _ty) x @ (value_type $B8))
+      (sext32_reg $I8 (put_in_reg x)))
+(rule (cast_bool (gpr32_ty _ty) x @ (value_type $B16))
+      (sext32_reg $I16 (put_in_reg x)))
+(rule (cast_bool (gpr64_ty _ty) x @ (value_type $B8))
+      (sext64_reg $I8 (put_in_reg x)))
+(rule (cast_bool (gpr64_ty _ty) x @ (value_type $B16))
+      (sext64_reg $I16 (put_in_reg x)))
+(rule (cast_bool (gpr64_ty _ty) x @ (value_type $B32))
+      (sext64_reg $I32 (put_in_reg x)))
+
+
+;;;; Rules for `bint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Mask with 1 to get a 0/1 result (8- or 16-bit types).
+(rule (lower (has_type (fits_in_16 ty) (bint x)))
+      (value_reg (and_uimm16shifted ty (put_in_reg x) (uimm16shifted 1 0))))
+
+;; Mask with 1 to get a 0/1 result (32-bit types).
+(rule (lower (has_type (fits_in_32 ty) (bint x)))
+      (value_reg (and_uimm32shifted ty (put_in_reg x) (uimm32shifted 1 0))))
+
+;; Mask with 1 to get a 0/1 result (64-bit types).
+(rule (lower (has_type (fits_in_64 ty) (bint x)))
+      (value_reg (and_reg ty (put_in_reg x) (imm ty 1))))
+
+
+;;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; The FLOGR hardware instruction always operates on the full 64-bit register.
+;; We can zero-extend smaller types, but then we have to compensate for the
+;; additional leading zero bits the instruction will actually see.
+(decl clz_offset (Type Reg) Reg)
+(rule (clz_offset $I8 x) (add_simm16 $I8 x -56))
+(rule (clz_offset $I16 x) (add_simm16 $I16 x -48))
+(rule (clz_offset $I32 x) (add_simm16 $I32 x -32))
+(rule (clz_offset $I64 x) (copy_reg $I64 x))
+
+;; Count leading zeros, via FLOGR on an input zero-extended to 64 bits,
+;; with the result compensated for the extra bits.
+(rule (lower (has_type (fits_in_64 ty) (clz x)))
+      (let ((ext_reg Reg (put_in_reg_zext64 x))
+            ;; Ask for a value of 64 in the all-zero 64-bit input case.
+            ;; After compensation this will match the expected semantics.
+            (clz RegPair (clz_reg 64 ext_reg)))
+        (value_reg (clz_offset ty (regpair_hi clz)))))
+
+
+;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Count leading sign-bit copies.  We don't have any instruction for that,
+;; so we instead count the leading zeros after inverting the input if negative,
+;; i.e. computing
+;;        cls(x) == clz(x ^ (x >> 63))
+;; where x is the sign-extended input.
+(rule (lower (has_type (fits_in_64 ty) (cls x)))
+      (let ((ext_reg Reg (put_in_reg_sext64 x))
+            (signbit_copies Reg (ashr_imm $I64 ext_reg 63))
+            (inv_reg Reg (xor_reg $I64 ext_reg signbit_copies))
+            (clz RegPair (clz_reg 64 inv_reg)))
+        (value_reg (clz_offset ty (regpair_hi clz)))))
+
+
+;;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; To count trailing zeros, we find the last bit set in the input via (x & -x),
+;; count the leading zeros of that value, and subtract from 63:
+;;
+;;   ctz(x) == 63 - clz(x & -x)
+;;
+;; This works for all cases except a zero input, where the above formula would
+;; return -1, but we are expected to return the type size.  The compensation
+;; for this case is handled differently for 64-bit types vs. smaller types.
+
+;; For smaller types, we simply ensure that the extended 64-bit input is
+;; never zero by setting a "guard bit" in the position corresponding to
+;; the input type size.  This way the 64-bit algorithm above will handle
+;; that case correctly automatically.
+(rule (lower (has_type (gpr32_ty ty) (ctz x)))
+      (let ((rx Reg (or_uimm16shifted $I64 (put_in_reg x) (ctz_guardbit ty)))
+            (lastbit Reg (and_reg $I64 rx (neg_reg $I64 rx)))
+            (clz RegPair (clz_reg 64 lastbit)))
+        (value_reg (sub_reg ty (imm ty 63) (regpair_hi clz)))))
+
+(decl ctz_guardbit (Type) UImm16Shifted)
+(rule (ctz_guardbit $I8) (uimm16shifted 256 0))
+(rule (ctz_guardbit $I16) (uimm16shifted 1 16))
+(rule (ctz_guardbit $I32) (uimm16shifted 1 32))
+
+;; For 64-bit types, the FLOGR instruction will indicate the zero input case
+;; via its condition code.  We check for that and replace the instruction
+;; result with the value -1 via a conditional move, which will then lead to
+;; the correct result after the final subtraction from 63.
+(rule (lower (has_type (gpr64_ty _ty) (ctz x)))
+      (let ((rx Reg (put_in_reg x))
+            (lastbit Reg (and_reg $I64 rx (neg_reg $I64 rx)))
+            (clz RegPair (clz_reg -1 lastbit)))
+        (value_reg (sub_reg $I64 (imm $I64 63) (regpair_hi clz)))))
+
+
+;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Population count for 8-bit types is supported by the POPCNT instruction.
+(rule (lower (has_type $I8 (popcnt x)))
+      (value_reg (popcnt_byte (put_in_reg x))))
+
+;; On z15, the POPCNT instruction has a variant to compute a full 64-bit
+;; population count, which we also use for 16- and 32-bit types.
+(rule (lower (has_type (and (mie2_enabled) (fits_in_64 ty)) (popcnt x)))
+      (value_reg (popcnt_reg (put_in_reg_zext64 x))))
+
+;; On z14, we use the regular POPCNT, which computes the population count
+;; of each input byte separately, so we need to accumulate those partial
+;; results via a series of log2(type size in bytes) - 1 additions.  We
+;; accumulate in the high byte, so that a final right shift will zero out
+;; any unrelated bits to give a clean result.
+
+(rule (lower (has_type (and (mie2_disabled) $I16) (popcnt x)))
+      (let ((cnt2 Reg (popcnt_byte (put_in_reg x)))
+            (cnt1 Reg (add_reg $I32 cnt2 (lshl_imm $I32 cnt2 8))))
+        (value_reg (lshr_imm $I32 cnt1 8))))
+
+(rule (lower (has_type (and (mie2_disabled) $I32) (popcnt x)))
+      (let ((cnt4 Reg (popcnt_byte (put_in_reg x)))
+            (cnt2 Reg (add_reg $I32 cnt4 (lshl_imm $I32 cnt4 16)))
+            (cnt1 Reg (add_reg $I32 cnt2 (lshl_imm $I32 cnt2 8))))
+        (value_reg (lshr_imm $I32 cnt1 24))))
+
+(rule (lower (has_type (and (mie2_disabled) $I64) (popcnt x)))
+      (let ((cnt8 Reg (popcnt_byte (put_in_reg x)))
+            (cnt4 Reg (add_reg $I64 cnt8 (lshl_imm $I64 cnt8 32)))
+            (cnt2 Reg (add_reg $I64 cnt4 (lshl_imm $I64 cnt4 16)))
+            (cnt1 Reg (add_reg $I64 cnt2 (lshl_imm $I64 cnt2 8))))
+        (value_reg (lshr_imm $I64 cnt1 56))))
+
+
+;;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add two registers.
+(rule (lower (has_type ty (fadd x y)))
+      (value_reg (fadd_reg ty (put_in_reg x) (put_in_reg y))))
+
+
+;;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Subtract two registers.
+(rule (lower (has_type ty (fsub x y)))
+      (value_reg (fsub_reg ty (put_in_reg x) (put_in_reg y))))
+
+
+;;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Multiply two registers.
+(rule (lower (has_type ty (fmul x y)))
+      (value_reg (fmul_reg ty (put_in_reg x) (put_in_reg y))))
+
+
+;;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Divide two registers.
+(rule (lower (has_type ty (fdiv x y)))
+      (value_reg (fdiv_reg ty (put_in_reg x) (put_in_reg y))))
+
+
+;;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Minimum of two registers.
+(rule (lower (has_type ty (fmin x y)))
+      (value_reg (fmin_reg ty (put_in_reg x) (put_in_reg y))))
+
+
+;;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Maximum of two registers.
+(rule (lower (has_type ty (fmax x y)))
+      (value_reg (fmax_reg ty (put_in_reg x) (put_in_reg y))))
+
+
+;;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Copysign of two registers.
+(rule (lower (has_type ty (fcopysign x y)))
+      (value_reg (fpu_copysign ty (put_in_reg x) (put_in_reg y))))
+
+
+;;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Multiply-and-add of three registers.
+(rule (lower (has_type ty (fma x y z)))
+      (value_reg (fma_reg ty (put_in_reg x) (put_in_reg y) (put_in_reg z))))
+
+
+;;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Square root of a register.
+(rule (lower (has_type ty (sqrt x)))
+      (value_reg (sqrt_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Negated value of a register.
+(rule (lower (has_type ty (fneg x)))
+      (value_reg (fneg_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Absolute value of a register.
+(rule (lower (has_type ty (fabs x)))
+      (value_reg (fabs_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `ceil` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Round value in a register towards positive infinity.
+(rule (lower (has_type ty (ceil x)))
+      (value_reg (ceil_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `floor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Round value in a register towards negative infinity.
+(rule (lower (has_type ty (floor x)))
+      (value_reg (floor_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `trunc` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Round value in a register towards zero.
+(rule (lower (has_type ty (trunc x)))
+      (value_reg (trunc_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `nearest` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Round value in a register towards nearest.
+(rule (lower (has_type ty (nearest x)))
+      (value_reg (nearest_reg ty (put_in_reg x))))
+
+
+;;;; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Promote a register.
+(rule (lower (has_type dst_ty (fpromote x @ (value_type src_ty))))
+      (value_reg (fpromote_reg dst_ty src_ty (put_in_reg x))))
+
+
+;;;; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Demote a register.
+(rule (lower (has_type dst_ty (fdemote x @ (value_type src_ty))))
+      (value_reg (fdemote_reg dst_ty src_ty (put_in_reg x))))
+
+
+;;;; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert an unsigned integer value in a register to floating-point.
+(rule (lower (has_type dst_ty (fcvt_from_uint x @ (value_type src_ty))))
+      (value_reg (fcvt_from_uint_reg dst_ty (ty_ext32 src_ty)
+                                            (put_in_reg_zext32 x))))
+
+
+;;;; Rules for `fcvt_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert a signed integer value in a register to floating-point.
+(rule (lower (has_type dst_ty (fcvt_from_sint x @ (value_type src_ty))))
+      (value_reg (fcvt_from_sint_reg dst_ty (ty_ext32 src_ty)
+                                            (put_in_reg_sext32 x))))
+
+
+;;;; Rules for `fcvt_to_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert a floating-point value in a register to an unsigned integer value.
+;; Traps if the input cannot be represented in the output type.
+;; FIXME: Add support for 8-/16-bit destination types (needs overflow check).
+(rule (lower (has_type (ty_32_or_64 dst_ty) (fcvt_to_uint x @ (value_type src_ty))))
+      (let ((src Reg (put_in_reg x))
+            ;; First, check whether the input is a NaN, and trap if so.
+            (_ Reg (trap_if (fcmp_reg src_ty src src)
+                            (floatcc_as_cond (FloatCC.Unordered))
+                            (trap_code_bad_conversion_to_integer)))
+            ;; Perform the conversion.  If this sets CC 3, we have a
+            ;; "special case".  Since we already exluded the case where
+            ;; the input was a NaN, the only other option is that the
+            ;; conversion overflowed the target type.
+            (dst Reg (trap_if (fcvt_to_uint_reg_with_flags dst_ty src_ty src)
+                              (floatcc_as_cond (FloatCC.Unordered))
+                              (trap_code_integer_overflow))))
+        (value_reg dst)))
+
+
+;;;; Rules for `fcvt_to_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert a floating-point value in a register to a signed integer value.
+;; Traps if the input cannot be represented in the output type.
+;; FIXME: Add support for 8-/16-bit destination types (needs overflow check).
+(rule (lower (has_type (ty_32_or_64 dst_ty) (fcvt_to_sint x @ (value_type src_ty))))
+      (let ((src Reg (put_in_reg x))
+            ;; First, check whether the input is a NaN, and trap if so.
+            (_ Reg (trap_if (fcmp_reg src_ty src src)
+                            (floatcc_as_cond (FloatCC.Unordered))
+                            (trap_code_bad_conversion_to_integer)))
+            ;; Perform the conversion.  If this sets CC 3, we have a
+            ;; "special case".  Since we already exluded the case where
+            ;; the input was a NaN, the only other option is that the
+            ;; conversion overflowed the target type.
+            (dst Reg (trap_if (fcvt_to_sint_reg_with_flags dst_ty src_ty src)
+                              (floatcc_as_cond (FloatCC.Unordered))
+                              (trap_code_integer_overflow))))
+        (value_reg dst)))
+
+
+;;;; Rules for `fcvt_to_uint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert a floating-point value in a register to an unsigned integer value.
+;; FIXME: Add support for 8-/16-bit destination types (needs overflow check).
+(rule (lower (has_type (ty_32_or_64 dst_ty) (fcvt_to_uint_sat x @ (value_type src_ty))))
+      (let ((src Reg (put_in_reg x))
+            (dst Reg (fcvt_to_uint_reg dst_ty src_ty src))
+            ;; In most special cases, the Z instruction already yields the
+            ;; result expected by Cranelift semantics.  The only exception
+            ;; it the case where the input was a NaN.  We explicitly check
+            ;; for that and force the output to 0 in that case.
+            (sat Reg (with_flags_1 (fcmp_reg src_ty src src)
+                                   (cmov_imm dst_ty
+                                     (floatcc_as_cond (FloatCC.Unordered)) 0 dst))))
+        (value_reg sat)))
+
+
+;;;; Rules for `fcvt_to_sint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert a floating-point value in a register to a signed integer value.
+;; FIXME: Add support for 8-/16-bit destination types (needs overflow check).
+(rule (lower (has_type (ty_32_or_64 dst_ty) (fcvt_to_sint_sat x @ (value_type src_ty))))
+      (let ((src Reg (put_in_reg x))
+            (dst Reg (fcvt_to_sint_reg dst_ty src_ty src))
+            ;; In most special cases, the Z instruction already yields the
+            ;; result expected by Cranelift semantics.  The only exception
+            ;; it the case where the input was a NaN.  We explicitly check
+            ;; for that and force the output to 0 in that case.
+            (sat Reg (with_flags_1 (fcmp_reg src_ty src src)
+                                   (cmov_imm dst_ty
+                                     (floatcc_as_cond (FloatCC.Unordered)) 0 dst))))
+        (value_reg sat)))
+
+
+;;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Reinterpret a 64-bit integer value as floating-point.
+(rule (lower (has_type $F64 (bitcast x @ (value_type $I64))))
+      (value_reg (mov_to_fpr (put_in_reg x))))
+
+;; Reinterpret a 64-bit floating-point value as integer.
+(rule (lower (has_type $I64 (bitcast x @ (value_type $F64))))
+      (value_reg (mov_from_fpr (put_in_reg x))))
+
+;; Reinterpret a 32-bit integer value as floating-point (via $I64).
+;; Note that a 32-bit float is located in the high bits of the GPR.
+(rule (lower (has_type $F32 (bitcast x @ (value_type $I32))))
+      (value_reg (mov_to_fpr (lshl_imm $I64 (put_in_reg x) 32))))
+
+;; Reinterpret a 32-bit floating-point value as integer (via $I64).
+;; Note that a 32-bit float is located in the high bits of the GPR.
+(rule (lower (has_type $I32 (bitcast x @ (value_type $F32))))
+      (value_reg (lshr_imm $I64 (mov_from_fpr (put_in_reg x)) 32)))
+
+
+;;;; Rules for `stack_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Load the address of a stack slot.
+(rule (lower (has_type ty (stack_addr stack_slot offset)))
+      (value_reg (stack_addr_impl ty stack_slot offset)))
+
+
+;;;; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Load the address of a function, target reachable via PC-relative instruction.
+(rule (lower (and (func_addr _)
+                  (call_target_data name (reloc_distance_near))))
+      (value_reg (load_addr (memarg_symbol name 0 (memflags_trusted)))))
+
+;; Load the address of a function, general case.
+(rule (lower (and (func_addr _)
+                  (call_target_data name _)))
+      (value_reg (load_ext_name_far name 0)))
+
+
+;;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Load the address of a symbol, target reachable via PC-relative instruction.
+(rule (lower (and (symbol_value _)
+                  (symbol_value_data name (reloc_distance_near)
+                                          (memarg_symbol_offset offset))))
+      (value_reg (load_addr (memarg_symbol name offset (memflags_trusted)))))
+
+;; Load the address of a symbol, general case.
+(rule (lower (and (symbol_value _)
+                  (symbol_value_data name _ offset)))
+      (value_reg (load_ext_name_far name offset)))
+
+
+;;;; Rules for `load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Load 8-bit integers.
+(rule (lower (has_type $I8 (load flags addr offset)))
+      (value_reg (zext32_mem $I8 (lower_address flags addr offset))))
+
+;; Load 16-bit big-endian integers.
+(rule (lower (has_type $I16 (load flags @ (bigendian) addr offset)))
+      (value_reg (zext32_mem $I16 (lower_address flags addr offset))))
+
+;; Load 16-bit little-endian integers.
+(rule (lower (has_type $I16 (load flags @ (littleendian) addr offset)))
+      (value_reg (loadrev16 (lower_address flags addr offset))))
+
+;; Load 32-bit big-endian integers.
+(rule (lower (has_type $I32 (load flags @ (bigendian) addr offset)))
+      (value_reg (load32 (lower_address flags addr offset))))
+
+;; Load 32-bit little-endian integers.
+(rule (lower (has_type $I32 (load flags @ (littleendian) addr offset)))
+      (value_reg (loadrev32 (lower_address flags addr offset))))
+
+;; Load 64-bit big-endian integers.
+(rule (lower (has_type $I64 (load flags @ (bigendian) addr offset)))
+      (value_reg (load64 (lower_address flags addr offset))))
+
+;; Load 64-bit little-endian integers.
+(rule (lower (has_type $I64 (load flags @ (littleendian) addr offset)))
+      (value_reg (loadrev64 (lower_address flags addr offset))))
+
+;; Load 64-bit big-endian references.
+(rule (lower (has_type $R64 (load flags @ (bigendian) addr offset)))
+      (value_reg (load64 (lower_address flags addr offset))))
+
+;; Load 64-bit little-endian references.
+(rule (lower (has_type $R64 (load flags @ (littleendian) addr offset)))
+      (value_reg (loadrev64 (lower_address flags addr offset))))
+
+;; Load 32-bit big-endian floating-point values.
+(rule (lower (has_type $F32 (load flags @ (bigendian) addr offset)))
+      (value_reg (fpu_load32 (lower_address flags addr offset))))
+
+;; Load 32-bit little-endian floating-point values (z15 instruction).
+(rule (lower (has_type (and (vxrs_ext2_enabled) $F32)
+                       (load flags @ (littleendian) addr offset)))
+      (value_reg (fpu_loadrev32 (lower_address flags addr offset))))
+
+;; Load 32-bit little-endian floating-point values (via GPR on z14).
+(rule (lower (has_type (and (vxrs_ext2_disabled) $F32)
+                       (load flags @ (littleendian) addr offset)))
+      (let ((gpr Reg (loadrev32 (lower_address flags addr offset))))
+        (value_reg (mov_to_fpr (lshl_imm $I64 gpr 32)))))
+
+;; Load 64-bit big-endian floating-point values.
+(rule (lower (has_type $F64 (load flags @ (bigendian) addr offset)))
+      (value_reg (fpu_load64 (lower_address flags addr offset))))
+
+;; Load 64-bit little-endian floating-point values (z15 instruction).
+(rule (lower (has_type (and (vxrs_ext2_enabled) $F64)
+                            (load flags @ (littleendian) addr offset)))
+      (value_reg (fpu_loadrev64 (lower_address flags addr offset))))
+
+;; Load 64-bit little-endian floating-point values (via GPR on z14).
+(rule (lower (has_type (and (vxrs_ext2_disabled) $F64)
+                            (load flags @ (littleendian) addr offset)))
+      (let ((gpr Reg (loadrev64 (lower_address flags addr offset))))
+        (value_reg (mov_to_fpr gpr))))
+
+
+;;;; Rules for `uload8` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 16- or 32-bit target types.
+(rule (lower (has_type (gpr32_ty _ty) (uload8 flags addr offset)))
+      (value_reg (zext32_mem $I8 (lower_address flags addr offset))))
+
+;; 64-bit target types.
+(rule (lower (has_type (gpr64_ty _ty) (uload8 flags addr offset)))
+      (value_reg (zext64_mem $I8 (lower_address flags addr offset))))
+
+
+;;;; Rules for `sload8` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 16- or 32-bit target types.
+(rule (lower (has_type (gpr32_ty _ty) (sload8 flags addr offset)))
+      (value_reg (sext32_mem $I8 (lower_address flags addr offset))))
+
+;; 64-bit target types.
+(rule (lower (has_type (gpr64_ty _ty) (sload8 flags addr offset)))
+      (value_reg (sext64_mem $I8 (lower_address flags addr offset))))
+
+
+;;;; Rules for `uload16` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 32-bit target type, big-endian source value.
+(rule (lower (has_type (gpr32_ty _ty)
+                       (uload16 flags @ (bigendian) addr offset)))
+      (value_reg (zext32_mem $I16 (lower_address flags addr offset))))
+
+;; 32-bit target type, little-endian source value (via explicit extension).
+(rule (lower (has_type (gpr32_ty _ty)
+                       (uload16 flags @ (littleendian) addr offset)))
+      (let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))
+        (value_reg (zext32_reg $I16 reg16))))
+
+;; 64-bit target type, big-endian source value.
+(rule (lower (has_type (gpr64_ty _ty)
+                       (uload16 flags @ (bigendian) addr offset)))
+      (value_reg (zext64_mem $I16 (lower_address flags addr offset))))
+
+;; 64-bit target type, little-endian source value (via explicit extension).
+(rule (lower (has_type (gpr64_ty _ty)
+                       (uload16 flags @ (littleendian) addr offset)))
+      (let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))
+        (value_reg (zext64_reg $I16 reg16))))
+
+
+;;;; Rules for `sload16` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 32-bit target type, big-endian source value.
+(rule (lower (has_type (gpr32_ty _ty)
+                       (sload16 flags @ (bigendian) addr offset)))
+      (value_reg (sext32_mem $I16 (lower_address flags addr offset))))
+
+;; 32-bit target type, little-endian source value (via explicit extension).
+(rule (lower (has_type (gpr32_ty _ty)
+                       (sload16 flags @ (littleendian) addr offset)))
+      (let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))
+        (value_reg (sext32_reg $I16 reg16))))
+
+;; 64-bit target type, big-endian source value.
+(rule (lower (has_type (gpr64_ty _ty)
+                       (sload16 flags @ (bigendian) addr offset)))
+      (value_reg (sext64_mem $I16 (lower_address flags addr offset))))
+
+;; 64-bit target type, little-endian source value (via explicit extension).
+(rule (lower (has_type (gpr64_ty _ty)
+                       (sload16 flags @ (littleendian) addr offset)))
+      (let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))
+        (value_reg (sext64_reg $I16 reg16))))
+
+
+;;;; Rules for `uload32` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 64-bit target type, big-endian source value.
+(rule (lower (has_type (gpr64_ty _ty)
+                       (uload32 flags @ (bigendian) addr offset)))
+      (value_reg (zext64_mem $I32 (lower_address flags addr offset))))
+
+;; 64-bit target type, little-endian source value (via explicit extension).
+(rule (lower (has_type (gpr64_ty _ty)
+                       (uload32 flags @ (littleendian) addr offset)))
+      (let ((reg32 Reg (loadrev32 (lower_address flags addr offset))))
+        (value_reg (zext64_reg $I32 reg32))))
+
+
+;;;; Rules for `sload32` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 64-bit target type, big-endian source value.
+(rule (lower (has_type (gpr64_ty _ty)
+                       (sload32 flags @ (bigendian) addr offset)))
+      (value_reg (sext64_mem $I32 (lower_address flags addr offset))))
+
+;; 64-bit target type, little-endian source value (via explicit extension).
+(rule (lower (has_type (gpr64_ty _ty)
+                       (sload32 flags @ (littleendian) addr offset)))
+      (let ((reg32 Reg (loadrev32 (lower_address flags addr offset))))
+        (value_reg (sext64_reg $I32 reg32))))
+
+
+;;;; Rules for `store` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; The actual store logic for integer types is identical for the `store`,
+;; `istoreNN`, and `atomic_store` instructions, so we share common helpers.
+
+;; Store 8-bit integer type, main lowering entry point.
+(rule (lower (store flags val @ (value_type $I8) addr offset))
+      (value_regs_none (istore8_impl flags val addr offset)))
+
+;; Store 16-bit integer type, main lowering entry point.
+(rule (lower (store flags val @ (value_type $I16) addr offset))
+      (value_regs_none (istore16_impl flags val addr offset)))
+
+;; Store 32-bit integer type, main lowering entry point.
+(rule (lower (store flags val @ (value_type $I32) addr offset))
+      (value_regs_none (istore32_impl flags val addr offset)))
+
+;; Store 64-bit integer type, main lowering entry point.
+(rule (lower (store flags val @ (value_type $I64) addr offset))
+      (value_regs_none (istore64_impl flags val addr offset)))
+
+;; Store 64-bit reference type, main lowering entry point.
+(rule (lower (store flags val @ (value_type $R64) addr offset))
+      (value_regs_none (istore64_impl flags val addr offset)))
+
+;; Store 32-bit big-endian floating-point type.
+(rule (lower (store flags @ (bigendian)
+                    val @ (value_type $F32) addr offset))
+      (value_regs_none (fpu_store32 (put_in_reg val)
+                                    (lower_address flags addr offset))))
+
+;; Store 32-bit little-endian floating-point type (z15 instruction).
+(rule (lower (store flags @ (littleendian)
+                    val @ (value_type (and $F32 (vxrs_ext2_enabled))) addr offset))
+      (value_regs_none (fpu_storerev32 (put_in_reg val)
+                                       (lower_address flags addr offset))))
+
+;; Store 32-bit little-endian floating-point type (via GPR on z14).
+(rule (lower (store flags @ (littleendian)
+                    val @ (value_type (and $F32 (vxrs_ext2_disabled))) addr offset))
+      (let ((gpr Reg (lshr_imm $I64 (mov_from_fpr (put_in_reg val)) 32)))
+        (value_regs_none (storerev32 gpr (lower_address flags addr offset)))))
+
+;; Store 64-bit big-endian floating-point type.
+(rule (lower (store flags @ (bigendian)
+                    val @ (value_type $F64) addr offset))
+      (value_regs_none (fpu_store64 (put_in_reg val)
+                                    (lower_address flags addr offset))))
+
+;; Store 64-bit little-endian floating-point type (z15 instruction).
+(rule (lower (store flags @ (littleendian)
+                    val @ (value_type (and $F64 (vxrs_ext2_enabled))) addr offset))
+      (value_regs_none (fpu_storerev64 (put_in_reg val)
+                                       (lower_address flags addr offset))))
+
+;; Store 64-bit little-endian floating-point type (via GPR on z14).
+(rule (lower (store flags @ (littleendian)
+                    val @ (value_type (and $F64 (vxrs_ext2_disabled))) addr offset))
+      (let ((gpr Reg (mov_from_fpr (put_in_reg val))))
+        (value_regs_none (storerev64 gpr (lower_address flags addr offset)))))
+
+
+;;;; Rules for 8-bit integer stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Main `istore8` lowering entry point, dispatching to the helper.
+(rule (lower (istore8 flags val addr offset))
+      (value_regs_none (istore8_impl flags val addr offset)))
+
+;; Helper to store 8-bit integer types.
+(decl istore8_impl (MemFlags Value Value Offset32) SideEffectNoResult)
+
+;; Store 8-bit integer types, register input.
+(rule (istore8_impl flags val addr offset)
+      (store8 (put_in_reg val) (lower_address flags addr offset)))
+
+;; Store 8-bit integer types, immediate input.
+(rule (istore8_impl flags (u8_from_value imm) addr offset)
+      (store8_imm imm (lower_address flags addr offset)))
+
+
+;;;; Rules for 16-bit integer stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Main `istore16` lowering entry point, dispatching to the helper.
+(rule (lower (istore16 flags val addr offset))
+      (value_regs_none (istore16_impl flags val addr offset)))
+
+;; Helper to store 16-bit integer types.
+(decl istore16_impl (MemFlags Value Value Offset32) SideEffectNoResult)
+
+;; Store 16-bit big-endian integer types, register input.
+(rule (istore16_impl flags @ (bigendian) val addr offset)
+      (store16 (put_in_reg val) (lower_address flags addr offset)))
+
+;; Store 16-bit little-endian integer types, register input.
+(rule (istore16_impl flags @ (littleendian) val addr offset)
+      (storerev16 (put_in_reg val) (lower_address flags addr offset)))
+
+;; Store 16-bit big-endian integer types, immediate input.
+(rule (istore16_impl flags @ (bigendian) (i16_from_value imm) addr offset)
+      (store16_imm imm (lower_address flags addr offset)))
+
+;; Store 16-bit little-endian integer types, immediate input.
+(rule (istore16_impl flags @ (littleendian) (i16_from_swapped_value imm) addr offset)
+      (store16_imm imm (lower_address flags addr offset)))
+
+
+;;;; Rules for 32-bit integer stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Main `istore32` lowering entry point, dispatching to the helper.
+(rule (lower (istore32 flags val addr offset))
+      (value_regs_none (istore32_impl flags val addr offset)))
+
+;; Helper to store 32-bit integer types.
+(decl istore32_impl (MemFlags Value Value Offset32) SideEffectNoResult)
+
+;; Store 32-bit big-endian integer types, register input.
+(rule (istore32_impl flags @ (bigendian) val addr offset)
+      (store32 (put_in_reg val) (lower_address flags addr offset)))
+
+;; Store 32-bit big-endian integer types, immediate input.
+(rule (istore32_impl flags @ (bigendian) (i16_from_value imm) addr offset)
+      (store32_simm16 imm (lower_address flags addr offset)))
+
+;; Store 32-bit little-endian integer types.
+(rule (istore32_impl flags @ (littleendian) val addr offset)
+      (storerev32 (put_in_reg val) (lower_address flags addr offset)))
+
+
+;;;; Rules for 64-bit integer stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Helper to store 64-bit integer types.
+(decl istore64_impl (MemFlags Value Value Offset32) SideEffectNoResult)
+
+;; Store 64-bit big-endian integer types, register input.
+(rule (istore64_impl flags @ (bigendian) val addr offset)
+      (store64 (put_in_reg val) (lower_address flags addr offset)))
+
+;; Store 64-bit big-endian integer types, immediate input.
+(rule (istore64_impl flags @ (bigendian) (i16_from_value imm) addr offset)
+      (store64_simm16 imm (lower_address flags addr offset)))
+
+;; Store 64-bit little-endian integer types.
+(rule (istore64_impl flags @ (littleendian) val addr offset)
+      (storerev64 (put_in_reg val) (lower_address flags addr offset)))
+
+
+;;;; Rules for `atomic_rmw` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Atomic AND for 32/64-bit big-endian types, using a single instruction.
+(rule (lower (has_type (ty_32_or_64 ty)
+                (atomic_rmw flags @ (bigendian) (AtomicRmwOp.And) addr src)))
+      (value_reg (atomic_rmw_and ty (put_in_reg src)
+                                 (lower_address flags addr (zero_offset)))))
+
+;; Atomic OR for 32/64-bit big-endian types, using a single instruction.
+(rule (lower (has_type (ty_32_or_64 ty)
+               (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Or) addr src)))
+      (value_reg (atomic_rmw_or ty (put_in_reg src)
+                                (lower_address flags addr (zero_offset)))))
+
+;; Atomic XOR for 32/64-bit big-endian types, using a single instruction.
+(rule (lower (has_type (ty_32_or_64 ty)
+               (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Xor) addr src)))
+      (value_reg (atomic_rmw_xor ty (put_in_reg src)
+                                 (lower_address flags addr (zero_offset)))))
+
+;; Atomic ADD for 32/64-bit big-endian types, using a single instruction.
+(rule (lower (has_type (ty_32_or_64 ty)
+               (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Add) addr src)))
+      (value_reg (atomic_rmw_add ty (put_in_reg src)
+                                 (lower_address flags addr (zero_offset)))))
+
+;; Atomic SUB for 32/64-bit big-endian types, using atomic ADD with negated input.
+(rule (lower (has_type (ty_32_or_64 ty)
+               (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Sub) addr src)))
+      (value_reg (atomic_rmw_add ty (neg_reg ty (put_in_reg src))
+                                 (lower_address flags addr (zero_offset)))))
+
+
+;;;; Rules for `atomic_cas` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; 32-bit big-endian atomic compare-and-swap instruction.
+(rule (lower (has_type $I32 (atomic_cas flags @ (bigendian) addr old new)))
+      (value_reg (atomic_cas32 (put_in_reg old) (put_in_reg new)
+                               (lower_address flags addr (zero_offset)))))
+
+;; 64-bit big-endian atomic compare-and-swap instruction.
+(rule (lower (has_type $I64 (atomic_cas flags @ (bigendian) addr old new)))
+      (value_reg (atomic_cas64 (put_in_reg old) (put_in_reg new)
+                               (lower_address flags addr (zero_offset)))))
+
+
+;;;; Rules for `atomic_load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Atomic loads can be implemented via regular loads on this platform.
+
+;; 8-bit atomic load.
+(rule (lower (has_type $I8 (atomic_load flags addr)))
+      (value_reg (zext32_mem $I8 (lower_address flags addr (zero_offset)))))
+
+;; 16-bit big-endian atomic load.
+(rule (lower (has_type $I16 (atomic_load flags @ (bigendian) addr)))
+      (value_reg (zext32_mem $I16 (lower_address flags addr (zero_offset)))))
+
+;; 16-bit little-endian atomic load.
+(rule (lower (has_type $I16 (atomic_load flags @ (littleendian) addr)))
+      (value_reg (loadrev16 (lower_address flags addr (zero_offset)))))
+
+;; 32-bit big-endian atomic load.
+(rule (lower (has_type $I32 (atomic_load flags @ (bigendian) addr)))
+      (value_reg (load32 (lower_address flags addr (zero_offset)))))
+
+;; 32-bit little-endian atomic load.
+(rule (lower (has_type $I32 (atomic_load flags @ (littleendian) addr)))
+      (value_reg (loadrev32 (lower_address flags addr (zero_offset)))))
+
+;; 64-bit big-endian atomic load.
+(rule (lower (has_type $I64 (atomic_load flags @ (bigendian) addr)))
+      (value_reg (load64 (lower_address flags addr (zero_offset)))))
+
+;; 64-bit little-endian atomic load.
+(rule (lower (has_type $I64 (atomic_load flags @ (littleendian) addr)))
+      (value_reg (loadrev64 (lower_address flags addr (zero_offset)))))
+
+
+;;;; Rules for `atomic_store` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Atomic stores can be implemented via regular stores followed by a fence.
+(decl atomic_store_impl (SideEffectNoResult) ValueRegs)
+(rule (atomic_store_impl store)
+      (let ((_ ValueRegs (value_regs_none store)))
+         (value_regs_none (fence_impl))))
+
+;; 8-bit atomic store.
+(rule (lower (atomic_store flags val @ (value_type $I8) addr))
+      (atomic_store_impl (istore8_impl flags val addr (zero_offset))))
+
+;; 16-bit atomic store.
+(rule (lower (atomic_store flags val @ (value_type $I16) addr))
+      (atomic_store_impl (istore16_impl flags val addr (zero_offset))))
+
+;; 32-bit atomic store.
+(rule (lower (atomic_store flags val @ (value_type $I32) addr))
+      (atomic_store_impl (istore32_impl flags val addr (zero_offset))))
+
+;; 64-bit atomic store.
+(rule (lower (atomic_store flags val @ (value_type $I64) addr))
+      (atomic_store_impl (istore64_impl flags val addr (zero_offset))))
+
+
+;;;; Rules for `fence` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Fence to ensure sequential consistency.
+(rule (lower (fence))
+      (value_regs_none (fence_impl)))
+
+
+;;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; We want to optimize the typical use of `icmp` (generating an integer 0/1
+;; result) followed by some user, like a `select` or a conditional branch.
+;; Instead of first generating the integer result and later testing it again,
+;; we want to sink the comparison to be performed at the site of use.
+;;
+;; To enable this, we provide generic helpers that return a `ProducesBool`
+;; encapsulating the comparison in question, which can be used by all the
+;; above scenarios.
+;;
+;; N.B. There are specific considerations when sinking a memory load into a
+;; comparison.  When emitting an `icmp` directly, this can of course be done
+;; as usual.  However, when we use the `ProducesBool` elsewhere, we need to
+;; consider *three* instructions: the load, the `icmp`, and the final user
+;; (e.g. a conditional branch).  The only way to safely sink the load would
+;; be to sink it direct into the final user, which is only possible if there
+;; is no *other* user of the `icmp` result.  This is not currently being
+;; verified by the `SinkableInst` logic, so to be safe we do not perform this
+;; optimization at all.
+;;
+;; The generic `icmp_val` helper therefore has a flag indicating whether
+;; it is being invoked in a context where it is safe to sink memory loads
+;; (e.g. when directly emitting an `icmp`), or whether it is not (e.g. when
+;; sinking the `icmp` result into a conditional branch or select).
+
+;; Main `icmp` entry point.  Generate a `ProducesBool` capturing the
+;; integer comparison and immediately lower it to a 0/1 integer result.
+;; In this case, it is safe to sink memory loads.
+(rule (lower (has_type ty (icmp int_cc x y)))
+      (value_reg (lower_bool ty (icmp_val $true int_cc x y))))
+
+
+;; Return a `ProducesBool` to implement any integer comparison.
+;; The first argument is a flag to indicate whether it is safe to sink
+;; memory loads as discussed above.
+(decl icmp_val (bool IntCC Value Value) ProducesBool)
+
+;; Dispatch for signed comparisons.
+(rule (icmp_val allow_mem int_cc @ (signed) x y)
+      (bool (icmps_val allow_mem x y) (intcc_as_cond int_cc)))
+;; Dispatch for unsigned comparisons.
+(rule (icmp_val allow_mem int_cc @ (unsigned) x y)
+      (bool (icmpu_val allow_mem x y) (intcc_as_cond int_cc)))
+
+
+;; Return a `ProducesBool` to implement signed integer comparisons.
+(decl icmps_val (bool Value Value) ProducesFlags)
+
+;; Compare (signed) two registers.
+(rule (icmps_val _ x @ (value_type (fits_in_64 ty)) y)
+      (icmps_reg (ty_ext32 ty) (put_in_reg_sext32 x) (put_in_reg_sext32 y)))
+
+;; Compare (signed) a register and a sign-extended register.
+(rule (icmps_val _ x @ (value_type (fits_in_64 ty)) (sext32_value y))
+      (icmps_reg_sext32 ty (put_in_reg x) (put_in_reg y)))
+
+;; Compare (signed) a register and an immediate.
+(rule (icmps_val _ x @ (value_type (fits_in_64 ty)) (i16_from_value y))
+      (icmps_simm16 (ty_ext32 ty) (put_in_reg_sext32 x) y))
+(rule (icmps_val _ x @ (value_type (fits_in_64 ty)) (i32_from_value y))
+      (icmps_simm32 (ty_ext32 ty) (put_in_reg_sext32 x) y))
+
+;; Compare (signed) a register and memory (32/64-bit types).
+(rule (icmps_val $true x @ (value_type (fits_in_64 ty)) (sinkable_load_32_64 y))
+      (icmps_mem ty (put_in_reg x) (sink_load y)))
+
+;; Compare (signed) a register and memory (16-bit types).
+(rule (icmps_val $true x @ (value_type (fits_in_64 ty)) (sinkable_load_16 y))
+      (icmps_mem_sext16 (ty_ext32 ty) (put_in_reg_sext32 x) (sink_load y)))
+
+;; Compare (signed) a register and sign-extended memory.
+(rule (icmps_val $true x @ (value_type (fits_in_64 ty)) (sinkable_sload16 y))
+      (icmps_mem_sext16 ty (put_in_reg x) (sink_sload16 y)))
+(rule (icmps_val $true x @ (value_type (fits_in_64 ty)) (sinkable_sload32 y))
+      (icmps_mem_sext32 ty (put_in_reg x) (sink_sload32 y)))
+
+
+;; Return a `ProducesBool` to implement unsigned integer comparisons.
+(decl icmpu_val (bool Value Value) ProducesFlags)
+
+;; Compare (unsigned) two registers.
+(rule (icmpu_val _ x @ (value_type (fits_in_64 ty)) y)
+      (icmpu_reg (ty_ext32 ty) (put_in_reg_zext32 x) (put_in_reg_zext32 y)))
+
+;; Compare (unsigned) a register and a sign-extended register.
+(rule (icmpu_val _ x @ (value_type (fits_in_64 ty)) (zext32_value y))
+      (icmpu_reg_zext32 ty (put_in_reg x) (put_in_reg y)))
+
+;; Compare (unsigned) a register and an immediate.
+(rule (icmpu_val _ x @ (value_type (fits_in_64 ty)) (u32_from_value y))
+      (icmpu_uimm32 (ty_ext32 ty) (put_in_reg_zext32 x) y))
+
+;; Compare (unsigned) a register and memory (32/64-bit types).
+(rule (icmpu_val $true x @ (value_type (fits_in_64 ty)) (sinkable_load_32_64 y))
+      (icmpu_mem ty (put_in_reg x) (sink_load y)))
+
+;; Compare (unsigned) a register and memory (16-bit types).
+;; Note that the ISA only provides instructions with a PC-relative memory
+;; address here, so we need to check whether the sinkable load matches this.
+(rule (icmpu_val $true x @ (value_type (fits_in_64 ty))
+                 (sinkable_load_16 (load_sym y)))
+      (icmpu_mem_zext16 (ty_ext32 ty) (put_in_reg_zext32 x) (sink_load y)))
+
+;; Compare (unsigned) a register and zero-extended memory.
+;; Note that the ISA only provides instructions with a PC-relative memory
+;; address here, so we need to check whether the sinkable load matches this.
+(rule (icmpu_val $true x @ (value_type (fits_in_64 ty))
+                 (sinkable_uload16 (uload16_sym y)))
+      (icmpu_mem_zext16 ty (put_in_reg x) (sink_uload16 y)))
+(rule (icmpu_val $true x @ (value_type (fits_in_64 ty)) (sinkable_uload32 y))
+      (icmpu_mem_zext32 ty (put_in_reg x) (sink_uload32 y)))
+
+
+;;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Main `fcmp` entry point.  Generate a `ProducesBool` capturing the
+;; integer comparison and immediately lower it to a 0/1 integer result.
+(rule (lower (has_type ty (fcmp float_cc x y)))
+      (value_reg (lower_bool ty (fcmp_val float_cc x y))))
+
+;; Return a `ProducesBool` to implement any floating-point comparison.
+(decl fcmp_val (FloatCC Value Value) ProducesBool)
+(rule (fcmp_val float_cc x @ (value_type ty) y)
+      (bool (fcmp_reg ty (put_in_reg x) (put_in_reg y))
+            (floatcc_as_cond float_cc)))
+
+
+;;;; Rules for `is_null` and `is_invalid`  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Null references are represented by the constant value 0.
+(rule (lower (has_type $B1 (is_null x @ (value_type $R64))))
+      (value_reg (lower_bool $B1 (bool (icmps_simm16 $I64 (put_in_reg x) 0)
+                                       (intcc_as_cond (IntCC.Equal))))))
+
+
+;; Invalid references are represented by the constant value -1.
+(rule (lower (has_type $B1 (is_invalid x @ (value_type $R64))))
+      (value_reg (lower_bool $B1 (bool (icmps_simm16 $I64 (put_in_reg x) -1)
+                                       (intcc_as_cond (IntCC.Equal))))))
+
+
+;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Return a `ProducesBool` to capture the fact that the input value is nonzero.
+;; In the common case where that input is the result of an `icmp` or `fcmp`
+;; instruction (possibly via an intermediate `bint`), directly use that compare.
+;; Note that it is not safe to sink memory loads here, see the `icmp` comment.
+(decl value_nonzero (Value) ProducesBool)
+(rule (value_nonzero (def_inst (bint val))) (value_nonzero val))
+(rule (value_nonzero (def_inst (icmp int_cc x y))) (icmp_val $false int_cc x y))
+(rule (value_nonzero (def_inst (fcmp float_cc x y))) (fcmp_val float_cc x y))
+(rule (value_nonzero val @ (value_type (gpr32_ty ty)))
+      (bool (icmps_simm16 $I32 (put_in_reg_sext32 val) 0)
+                          (intcc_as_cond (IntCC.NotEqual))))
+(rule (value_nonzero val @ (value_type (gpr64_ty ty)))
+      (bool (icmps_simm16 $I64 (put_in_reg val) 0)
+                          (intcc_as_cond (IntCC.NotEqual))))
+
+;; Main `select` entry point.  Lower the `value_nonzero` result.
+(rule (lower (has_type ty (select val_cond val_true val_false)))
+      (value_reg (select_bool_reg ty (value_nonzero val_cond)
+                                  (put_in_reg val_true) (put_in_reg val_false))))
+
+
+;;;; Rules for `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; We do not support the `iflags` mechanism on our platform.  However, common
+;; code will unconditionally emit certain patterns using `iflags` which we
+;; need to handle somehow.  Note that only those specific patterns are
+;; recognized by the code below, other uses will fail to lower.
+
+(rule (lower (has_type ty (selectif_spectre_guard int_cc
+                             (def_inst (ifcmp x y)) val_true val_false)))
+      (value_reg (select_bool_reg ty (icmp_val $false int_cc x y)
+                                  (put_in_reg val_true) (put_in_reg val_false))))
+
diff --git a/cranelift/codegen/src/isa/s390x/lower.rs b/cranelift/codegen/src/isa/s390x/lower.rs
index f4aa7c805d7c..866583922a30 100644
--- a/cranelift/codegen/src/isa/s390x/lower.rs
+++ b/cranelift/codegen/src/isa/s390x/lower.rs
@@ -1,8 +1,8 @@
 //! Lowering rules for S390x.
 
-use crate::ir::condcodes::{FloatCC, IntCC};
+use crate::ir::condcodes::IntCC;
 use crate::ir::Inst as IRInst;
-use crate::ir::{types, Endianness, InstructionData, MemFlags, Opcode, TrapCode, Type};
+use crate::ir::{types, Endianness, MemFlags, Opcode, Type};
 use crate::isa::s390x::abi::*;
 use crate::isa::s390x::inst::*;
 use crate::isa::s390x::settings as s390x_settings;
@@ -17,30 +17,11 @@ use core::convert::TryFrom;
 use regalloc::{Reg, Writable};
 use smallvec::SmallVec;
 
+pub mod isle;
+
 //=============================================================================
 // Helpers for instruction lowering.
 
-fn ty_is_int(ty: Type) -> bool {
-    match ty {
-        types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
-        types::I8 | types::I16 | types::I32 | types::I64 | types::R64 => true,
-        types::F32 | types::F64 => false,
-        types::IFLAGS | types::FFLAGS => panic!("Unexpected flags type"),
-        _ => panic!("ty_is_int() on unknown type: {:?}", ty),
-    }
-}
-
-fn ty_is_float(ty: Type) -> bool {
-    !ty_is_int(ty)
-}
-
-fn is_valid_atomic_transaction_ty(ty: Type) -> bool {
-    match ty {
-        types::I8 | types::I16 | types::I32 | types::I64 => true,
-        _ => false,
-    }
-}
-
 fn choose_32_64<T: Copy>(ty: Type, op32: T, op64: T) -> T {
     let bits = ty_bits(ty);
     if bits <= 32 {
@@ -71,22 +52,6 @@ fn input_matches_sconst<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) ->
     }
 }
 
-/// Return false if instruction input cannot have the value Imm, true otherwise.
-fn input_maybe_imm<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput, imm: u64) -> bool {
-    if let Some(c) = input_matches_const(ctx, input) {
-        let ty = ctx.input_ty(input.insn, input.input);
-        let from_bits = ty_bits(ty) as u8;
-        let mask = if from_bits < 64 {
-            (1u64 << ty_bits(ty)) - 1
-        } else {
-            0xffff_ffff_ffff_ffff
-        };
-        c & mask == imm & mask
-    } else {
-        true
-    }
-}
-
 /// Lower an instruction input to a 16-bit signed constant, if possible.
 fn input_matches_simm16<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<i16> {
     if let Some(imm_value) = input_matches_sconst(ctx, input) {
@@ -117,80 +82,6 @@ fn input_matches_uimm32<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) ->
     None
 }
 
-/// Lower a negated instruction input to a 16-bit signed constant, if possible.
-fn negated_input_matches_simm16<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-) -> Option<i16> {
-    if let Some(imm_value) = input_matches_sconst(ctx, input) {
-        if let Ok(imm) = i16::try_from(-imm_value) {
-            return Some(imm);
-        }
-    }
-    None
-}
-
-/// Lower a negated instruction input to a 32-bit signed constant, if possible.
-fn negated_input_matches_simm32<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-) -> Option<i32> {
-    if let Some(imm_value) = input_matches_sconst(ctx, input) {
-        if let Ok(imm) = i32::try_from(-imm_value) {
-            return Some(imm);
-        }
-    }
-    None
-}
-
-/// Lower an instruction input to a 16-bit shifted constant, if possible.
-fn input_matches_uimm16shifted<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-) -> Option<UImm16Shifted> {
-    if let Some(imm_value) = input_matches_const(ctx, input) {
-        return UImm16Shifted::maybe_from_u64(imm_value);
-    }
-    None
-}
-
-/// Lower an instruction input to a 32-bit shifted constant, if possible.
-fn input_matches_uimm32shifted<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-) -> Option<UImm32Shifted> {
-    if let Some(imm_value) = input_matches_const(ctx, input) {
-        return UImm32Shifted::maybe_from_u64(imm_value);
-    }
-    None
-}
-
-/// Lower an instruction input to a 16-bit inverted shifted constant, if possible.
-fn input_matches_uimm16shifted_inv<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-) -> Option<UImm16Shifted> {
-    if let Some(imm_value) = input_matches_const(ctx, input) {
-        if let Some(imm) = UImm16Shifted::maybe_from_u64(!imm_value) {
-            return Some(imm.negate_bits());
-        }
-    }
-    None
-}
-
-/// Lower an instruction input to a 32-bit inverted shifted constant, if possible.
-fn input_matches_uimm32shifted_inv<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    input: InsnInput,
-) -> Option<UImm32Shifted> {
-    if let Some(imm_value) = input_matches_const(ctx, input) {
-        if let Some(imm) = UImm32Shifted::maybe_from_u64(!imm_value) {
-            return Some(imm.negate_bits());
-        }
-    }
-    None
-}
-
 /// Checks for an instance of `op` feeding the given input.
 fn input_matches_insn<C: LowerCtx<I = Inst>>(
     c: &mut C,
@@ -545,84 +436,6 @@ fn lower_constant_u64<C: LowerCtx<I = Inst>>(ctx: &mut C, rd: Writable<Reg>, val
     }
 }
 
-fn lower_constant_u32<C: LowerCtx<I = Inst>>(ctx: &mut C, rd: Writable<Reg>, value: u32) {
-    for inst in Inst::load_constant32(rd, value) {
-        ctx.emit(inst);
-    }
-}
-
-fn lower_constant_f32<C: LowerCtx<I = Inst>>(ctx: &mut C, rd: Writable<Reg>, value: f32) {
-    ctx.emit(Inst::load_fp_constant32(rd, value));
-}
-
-fn lower_constant_f64<C: LowerCtx<I = Inst>>(ctx: &mut C, rd: Writable<Reg>, value: f64) {
-    ctx.emit(Inst::load_fp_constant64(rd, value));
-}
-
-//============================================================================
-// Lowering: miscellaneous helpers.
-
-/// Emit code to invert the value of type ty in register rd.
-fn lower_bnot<C: LowerCtx<I = Inst>>(ctx: &mut C, ty: Type, rd: Writable<Reg>) {
-    let alu_op = choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64);
-    ctx.emit(Inst::AluRUImm32Shifted {
-        alu_op,
-        rd,
-        imm: UImm32Shifted::maybe_from_u64(0xffff_ffff).unwrap(),
-    });
-    if ty_bits(ty) > 32 {
-        ctx.emit(Inst::AluRUImm32Shifted {
-            alu_op,
-            rd,
-            imm: UImm32Shifted::maybe_from_u64(0xffff_ffff_0000_0000).unwrap(),
-        });
-    }
-}
-
-/// Emit code to bitcast between integer and floating-point values.
-fn lower_bitcast<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    rd: Writable<Reg>,
-    output_ty: Type,
-    rn: Reg,
-    input_ty: Type,
-) {
-    match (input_ty, output_ty) {
-        (types::I64, types::F64) => {
-            ctx.emit(Inst::MovToFpr { rd, rn });
-        }
-        (types::F64, types::I64) => {
-            ctx.emit(Inst::MovFromFpr { rd, rn });
-        }
-        (types::I32, types::F32) => {
-            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-            ctx.emit(Inst::ShiftRR {
-                shift_op: ShiftOp::LShL64,
-                rd: tmp,
-                rn,
-                shift_imm: 32,
-                shift_reg: zero_reg(),
-            });
-            ctx.emit(Inst::MovToFpr {
-                rd,
-                rn: tmp.to_reg(),
-            });
-        }
-        (types::F32, types::I32) => {
-            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-            ctx.emit(Inst::MovFromFpr { rd: tmp, rn });
-            ctx.emit(Inst::ShiftRR {
-                shift_op: ShiftOp::LShR64,
-                rd,
-                rn: tmp.to_reg(),
-                shift_imm: 32,
-                shift_reg: zero_reg(),
-            });
-        }
-        _ => unreachable!("invalid bitcast from {:?} to {:?}", input_ty, output_ty),
-    }
-}
-
 //=============================================================================
 // Lowering: comparisons
 
@@ -792,24 +605,6 @@ fn lower_boolean_to_flags<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput)
     }
 }
 
-fn lower_flags_to_bool_result<C: LowerCtx<I = Inst>>(
-    ctx: &mut C,
-    cond: Cond,
-    rd: Writable<Reg>,
-    ty: Type,
-) {
-    if ty_bits(ty) == 1 {
-        lower_constant_u32(ctx, rd, 0);
-        ctx.emit(Inst::CMov32SImm16 { rd, cond, imm: 1 });
-    } else if ty_bits(ty) < 64 {
-        lower_constant_u32(ctx, rd, 0);
-        ctx.emit(Inst::CMov32SImm16 { rd, cond, imm: -1 });
-    } else {
-        lower_constant_u64(ctx, rd, 0);
-        ctx.emit(Inst::CMov64SImm16 { rd, cond, imm: -1 });
-    }
-}
-
 //============================================================================
 // Lowering: main entry point for lowering a instruction
 
@@ -832,1759 +627,156 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         None
     };
 
-    match op {
-        Opcode::Nop => {
-            // Nothing.
-        }
-
-        Opcode::Copy | Opcode::Ireduce | Opcode::Breduce => {
-            // Smaller ints / bools have the high bits undefined, so any reduce
-            // operation is simply a copy.
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let ty = ctx.input_ty(insn, 0);
-            ctx.emit(Inst::gen_move(rd, rn, ty));
-        }
+    if let Ok(()) = super::lower::isle::lower(ctx, flags, isa_flags, &outputs, insn) {
+        return Ok(());
+    }
 
-        Opcode::Iconst | Opcode::Bconst | Opcode::Null => {
-            let value = ctx.get_constant(insn).unwrap();
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let ty = ty.unwrap();
-            if ty.bits() <= 32 {
-                lower_constant_u32(ctx, rd, value as u32);
-            } else {
-                lower_constant_u64(ctx, rd, value);
-            }
-        }
-        Opcode::F32const => {
-            let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            lower_constant_f32(ctx, rd, value);
-        }
-        Opcode::F64const => {
-            let value = f64::from_bits(ctx.get_constant(insn).unwrap());
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            lower_constant_f64(ctx, rd, value);
-        }
+    let implemented_in_isle = || {
+        unreachable!(
+            "implemented in ISLE: inst = `{}`, type = `{:?}`",
+            ctx.dfg().display_inst(insn),
+            ty
+        );
+    };
 
-        Opcode::Iadd => {
-            let ty = ty.unwrap();
-            let alu_op = choose_32_64(ty, ALUOp::Add32, ALUOp::Add64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if let Some(imm) = input_matches_simm16(ctx, inputs[1]) {
-                ctx.emit(Inst::AluRRSImm16 {
-                    alu_op,
-                    rd,
-                    rn,
-                    imm,
-                });
-            } else if let Some(imm) = input_matches_simm32(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRSImm32 { alu_op, rd, imm });
-            } else if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else if let Some(mem) = input_matches_sext16_mem(ctx, inputs[1]) {
-                let alu_op = choose_32_64(ty, ALUOp::Add32Ext16, ALUOp::Add64Ext16);
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else if let Some(mem) = input_matches_sext32_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX {
-                    alu_op: ALUOp::Add64Ext32,
-                    rd,
-                    mem,
-                });
-            } else if let Some(rm) = input_matches_sext32_reg(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRR {
-                    alu_op: ALUOp::Add64Ext32,
-                    rd,
-                    rm,
-                });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            }
-        }
-        Opcode::Isub => {
-            let ty = ty.unwrap();
-            let alu_op = choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64);
-            let neg_op = choose_32_64(ty, ALUOp::Add32, ALUOp::Add64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if let Some(imm) = negated_input_matches_simm16(ctx, inputs[1]) {
-                ctx.emit(Inst::AluRRSImm16 {
-                    alu_op: neg_op,
-                    rd,
-                    rn,
-                    imm,
-                });
-            } else if let Some(imm) = negated_input_matches_simm32(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRSImm32 {
-                    alu_op: neg_op,
-                    rd,
-                    imm,
-                });
-            } else if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else if let Some(mem) = input_matches_sext16_mem(ctx, inputs[1]) {
-                let alu_op = choose_32_64(ty, ALUOp::Sub32Ext16, ALUOp::Sub64Ext16);
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else if let Some(mem) = input_matches_sext32_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX {
-                    alu_op: ALUOp::Sub64Ext32,
-                    rd,
-                    mem,
-                });
-            } else if let Some(rm) = input_matches_sext32_reg(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRR {
-                    alu_op: ALUOp::Sub64Ext32,
-                    rd,
-                    rm,
-                });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            }
-        }
-        Opcode::IaddIfcout => {
-            let ty = ty.unwrap();
-            assert!(ty == types::I32 || ty == types::I64);
-            // Emit an ADD LOGICAL instruction, which sets the condition code
-            // to indicate an (unsigned) carry bit.
-            let alu_op = choose_32_64(ty, ALUOp::AddLogical32, ALUOp::AddLogical64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if let Some(imm) = input_matches_uimm32(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm32 { alu_op, rd, imm });
-            } else if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else if let Some(mem) = input_matches_uext32_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX {
-                    alu_op: ALUOp::AddLogical64Ext32,
-                    rd,
-                    mem,
-                });
-            } else if let Some(rm) = input_matches_uext32_reg(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRR {
-                    alu_op: ALUOp::AddLogical64Ext32,
-                    rd,
-                    rm,
-                });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            }
-        }
+    match op {
+        Opcode::Nop
+        | Opcode::Copy
+        | Opcode::Iconst
+        | Opcode::Bconst
+        | Opcode::F32const
+        | Opcode::F64const
+        | Opcode::Null
+        | Opcode::Iadd
+        | Opcode::IaddIfcout
+        | Opcode::Isub
+        | Opcode::Iabs
+        | Opcode::Ineg
+        | Opcode::Imul
+        | Opcode::Umulhi
+        | Opcode::Smulhi
+        | Opcode::Udiv
+        | Opcode::Urem
+        | Opcode::Sdiv
+        | Opcode::Srem
+        | Opcode::Ishl
+        | Opcode::Ushr
+        | Opcode::Sshr
+        | Opcode::Rotr
+        | Opcode::Rotl
+        | Opcode::Ireduce
+        | Opcode::Uextend
+        | Opcode::Sextend
+        | Opcode::Bnot
+        | Opcode::Band
+        | Opcode::Bor
+        | Opcode::Bxor
+        | Opcode::BandNot
+        | Opcode::BorNot
+        | Opcode::BxorNot
+        | Opcode::Bitselect
+        | Opcode::Breduce
+        | Opcode::Bextend
+        | Opcode::Bmask
+        | Opcode::Bint
+        | Opcode::Clz
+        | Opcode::Cls
+        | Opcode::Ctz
+        | Opcode::Popcnt
+        | Opcode::Fadd
+        | Opcode::Fsub
+        | Opcode::Fmul
+        | Opcode::Fdiv
+        | Opcode::Fmin
+        | Opcode::Fmax
+        | Opcode::Sqrt
+        | Opcode::Fneg
+        | Opcode::Fabs
+        | Opcode::Fpromote
+        | Opcode::Fdemote
+        | Opcode::Ceil
+        | Opcode::Floor
+        | Opcode::Trunc
+        | Opcode::Nearest
+        | Opcode::Fma
+        | Opcode::Fcopysign
+        | Opcode::FcvtFromUint
+        | Opcode::FcvtFromSint
+        | Opcode::FcvtToUint
+        | Opcode::FcvtToSint
+        | Opcode::FcvtToUintSat
+        | Opcode::FcvtToSintSat
+        | Opcode::Bitcast
+        | Opcode::Load
+        | Opcode::Uload8
+        | Opcode::Sload8
+        | Opcode::Uload16
+        | Opcode::Sload16
+        | Opcode::Uload32
+        | Opcode::Sload32
+        | Opcode::Store
+        | Opcode::Istore8
+        | Opcode::Istore16
+        | Opcode::Istore32
+        | Opcode::AtomicRmw
+        | Opcode::AtomicCas
+        | Opcode::AtomicLoad
+        | Opcode::AtomicStore
+        | Opcode::Fence
+        | Opcode::Icmp
+        | Opcode::Fcmp
+        | Opcode::IsNull
+        | Opcode::IsInvalid
+        | Opcode::Select
+        | Opcode::SelectifSpectreGuard
+        | Opcode::StackAddr
+        | Opcode::FuncAddr
+        | Opcode::SymbolValue => implemented_in_isle(),
 
         Opcode::UaddSat | Opcode::SaddSat => unimplemented!(),
         Opcode::UsubSat | Opcode::SsubSat => unimplemented!(),
 
-        Opcode::Iabs => {
-            let ty = ty.unwrap();
-            let op = choose_32_64(ty, UnaryOp::Abs32, UnaryOp::Abs64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            if let Some(rn) = input_matches_sext32_reg(ctx, inputs[0]) {
-                ctx.emit(Inst::UnaryRR {
-                    op: UnaryOp::Abs64Ext32,
-                    rd,
-                    rn,
-                });
-            } else {
-                let narrow_mode = if ty.bits() < 32 {
-                    NarrowValueMode::SignExtend32
-                } else {
-                    NarrowValueMode::None
-                };
-                let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-                ctx.emit(Inst::UnaryRR { op, rd, rn });
-            }
-        }
-        Opcode::Ineg => {
-            let ty = ty.unwrap();
-            let op = choose_32_64(ty, UnaryOp::Neg32, UnaryOp::Neg64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            if let Some(rn) = input_matches_sext32_reg(ctx, inputs[0]) {
-                ctx.emit(Inst::UnaryRR {
-                    op: UnaryOp::Neg64Ext32,
-                    rd,
-                    rn,
-                });
-            } else {
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                ctx.emit(Inst::UnaryRR { op, rd, rn });
-            }
-        }
+        Opcode::Bitrev => unimplemented!(),
 
-        Opcode::Imul => {
-            let ty = ty.unwrap();
-            let alu_op = choose_32_64(ty, ALUOp::Mul32, ALUOp::Mul64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if let Some(imm) = input_matches_simm16(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRSImm16 { alu_op, rd, imm });
-            } else if let Some(imm) = input_matches_simm32(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRSImm32 { alu_op, rd, imm });
-            } else if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else if let Some(mem) = input_matches_sext16_mem(ctx, inputs[1]) {
-                let alu_op = choose_32_64(ty, ALUOp::Mul32Ext16, ALUOp::Mul64Ext16);
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else if let Some(mem) = input_matches_sext32_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX {
-                    alu_op: ALUOp::Mul64Ext32,
-                    rd,
-                    mem,
-                });
-            } else if let Some(rm) = input_matches_sext32_reg(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRR {
-                    alu_op: ALUOp::Mul64Ext32,
-                    rd,
-                    rm,
-                });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            }
-        }
+        Opcode::FcvtLowFromSint => unimplemented!("FcvtLowFromSint"),
 
-        Opcode::Umulhi | Opcode::Smulhi => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let is_signed = op == Opcode::Smulhi;
-            let input_ty = ctx.input_ty(insn, 0);
-            assert!(ctx.input_ty(insn, 1) == input_ty);
-            assert!(ctx.output_ty(insn, 0) == input_ty);
-
-            match input_ty {
-                types::I64 => {
-                    let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                    let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-
-                    if is_signed {
-                        ctx.emit(Inst::SMulWide { rn, rm });
-                        ctx.emit(Inst::gen_move(rd, gpr(0), input_ty));
-                    } else {
-                        ctx.emit(Inst::gen_move(writable_gpr(1), rm, input_ty));
-                        ctx.emit(Inst::UMulWide { rn });
-                        ctx.emit(Inst::gen_move(rd, gpr(0), input_ty));
-                    }
-                }
-                types::I32 => {
-                    let narrow_mode = if is_signed {
-                        NarrowValueMode::SignExtend64
-                    } else {
-                        NarrowValueMode::ZeroExtend64
-                    };
-                    let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-                    let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
-                    ctx.emit(Inst::AluRRR {
-                        alu_op: ALUOp::Mul64,
-                        rd,
-                        rn,
-                        rm,
-                    });
-                    let shift_op = if is_signed {
-                        ShiftOp::AShR64
-                    } else {
-                        ShiftOp::LShR64
-                    };
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op,
-                        rd,
-                        rn: rd.to_reg(),
-                        shift_imm: 32,
-                        shift_reg: zero_reg(),
-                    });
-                }
-                types::I16 | types::I8 => {
-                    let narrow_mode = if is_signed {
-                        NarrowValueMode::SignExtend32
-                    } else {
-                        NarrowValueMode::ZeroExtend32
-                    };
-                    let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-                    let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
-                    ctx.emit(Inst::AluRRR {
-                        alu_op: ALUOp::Mul32,
-                        rd,
-                        rn,
-                        rm,
-                    });
-                    let shift_op = if is_signed {
-                        ShiftOp::AShR32
-                    } else {
-                        ShiftOp::LShR32
-                    };
-                    let shift_amt = match input_ty {
-                        types::I16 => 16,
-                        types::I8 => 8,
-                        _ => unreachable!(),
-                    };
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op,
-                        rd,
-                        rn: rd.to_reg(),
-                        shift_imm: shift_amt,
-                        shift_reg: zero_reg(),
-                    });
-                }
-                _ => {
-                    panic!("Unsupported argument type for umulhi/smulhi: {}", input_ty);
-                }
-            }
+        Opcode::StackLoad | Opcode::StackStore => {
+            panic!("Direct stack memory access not supported; should not be used by Wasm");
         }
 
-        Opcode::Udiv | Opcode::Urem => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let ty = ty.unwrap();
-
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if ty_bits(ty) <= 32 {
-                lower_constant_u32(ctx, writable_gpr(0), 0);
-                if ty_bits(ty) < 32 {
-                    ctx.emit(Inst::Extend {
-                        rd: writable_gpr(1),
-                        rn,
-                        signed: false,
-                        from_bits: ty_bits(ty) as u8,
-                        to_bits: 32,
-                    });
-                } else {
-                    ctx.emit(Inst::mov32(writable_gpr(1), rn));
-                }
-            } else {
-                lower_constant_u64(ctx, writable_gpr(0), 0);
-                ctx.emit(Inst::mov64(writable_gpr(1), rn));
-            }
-
-            let narrow_mode = if ty.bits() < 32 {
-                NarrowValueMode::ZeroExtend32
-            } else {
-                NarrowValueMode::None
-            };
-            let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
-
-            if input_maybe_imm(ctx, inputs[1], 0) && flags.avoid_div_traps() {
-                ctx.emit(Inst::CmpTrapRSImm16 {
-                    op: choose_32_64(ty, CmpOp::CmpS32, CmpOp::CmpS64),
-                    rn: rm,
-                    imm: 0,
-                    cond: Cond::from_intcc(IntCC::Equal),
-                    trap_code: TrapCode::IntegerDivisionByZero,
-                });
-            }
-
-            if ty_bits(ty) <= 32 {
-                ctx.emit(Inst::UDivMod32 { rn: rm });
-            } else {
-                ctx.emit(Inst::UDivMod64 { rn: rm });
-            }
+        Opcode::ConstAddr => unimplemented!(),
 
-            if op == Opcode::Udiv {
-                ctx.emit(Inst::gen_move(rd, gpr(1), ty));
-            } else {
-                ctx.emit(Inst::gen_move(rd, gpr(0), ty));
-            }
+        Opcode::HeapAddr => {
+            panic!("heap_addr should have been removed by legalization!");
         }
 
-        Opcode::Sdiv | Opcode::Srem => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let ty = ty.unwrap();
-
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if ty_bits(ty) < 64 {
-                ctx.emit(Inst::Extend {
-                    rd: writable_gpr(1),
-                    rn,
-                    signed: true,
-                    from_bits: ty_bits(ty) as u8,
-                    to_bits: 64,
-                });
-            } else {
-                ctx.emit(Inst::mov64(writable_gpr(1), rn));
-            }
-
-            let narrow_mode = if ty.bits() < 32 {
-                NarrowValueMode::SignExtend32
-            } else {
-                NarrowValueMode::None
-            };
-            let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
-
-            if input_maybe_imm(ctx, inputs[1], 0) && flags.avoid_div_traps() {
-                ctx.emit(Inst::CmpTrapRSImm16 {
-                    op: choose_32_64(ty, CmpOp::CmpS32, CmpOp::CmpS64),
-                    rn: rm,
-                    imm: 0,
-                    cond: Cond::from_intcc(IntCC::Equal),
-                    trap_code: TrapCode::IntegerDivisionByZero,
-                });
-            }
-
-            if input_maybe_imm(ctx, inputs[1], 0xffff_ffff_ffff_ffff) {
-                if op == Opcode::Sdiv {
-                    let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
-                    if ty_bits(ty) <= 32 {
-                        lower_constant_u32(ctx, tmp, (1 << (ty_bits(ty) - 1)) - 1);
-                    } else {
-                        lower_constant_u64(ctx, tmp, (1 << (ty_bits(ty) - 1)) - 1);
-                    }
-                    ctx.emit(Inst::AluRRR {
-                        alu_op: choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64),
-                        rd: tmp,
-                        rn: tmp.to_reg(),
-                        rm: gpr(1),
-                    });
-                    ctx.emit(Inst::AluRRR {
-                        alu_op: choose_32_64(ty, ALUOp::And32, ALUOp::And64),
-                        rd: tmp,
-                        rn: tmp.to_reg(),
-                        rm,
-                    });
-                    ctx.emit(Inst::CmpTrapRSImm16 {
-                        op: choose_32_64(ty, CmpOp::CmpS32, CmpOp::CmpS64),
-                        rn: tmp.to_reg(),
-                        imm: -1,
-                        cond: Cond::from_intcc(IntCC::Equal),
-                        trap_code: TrapCode::IntegerOverflow,
-                    });
-                } else {
-                    if ty_bits(ty) > 32 {
-                        ctx.emit(Inst::CmpRSImm16 {
-                            op: CmpOp::CmpS64,
-                            rn: rm,
-                            imm: -1,
-                        });
-                        ctx.emit(Inst::CMov64SImm16 {
-                            rd: writable_gpr(1),
-                            cond: Cond::from_intcc(IntCC::Equal),
-                            imm: 0,
-                        });
-                    }
-                }
-            }
-
-            if ty_bits(ty) <= 32 {
-                ctx.emit(Inst::SDivMod32 { rn: rm });
-            } else {
-                ctx.emit(Inst::SDivMod64 { rn: rm });
-            }
-
-            if op == Opcode::Sdiv {
-                ctx.emit(Inst::gen_move(rd, gpr(1), ty));
-            } else {
-                ctx.emit(Inst::gen_move(rd, gpr(0), ty));
-            }
+        Opcode::TableAddr => {
+            panic!("table_addr should have been removed by legalization!");
         }
 
-        Opcode::Uextend | Opcode::Sextend => {
-            let ty = ty.unwrap();
-            let to_bits = ty_bits(ty) as u8;
-            let to_bits = std::cmp::max(32, to_bits);
-            let narrow_mode = match (op, to_bits) {
-                (Opcode::Uextend, 32) => NarrowValueMode::ZeroExtend32,
-                (Opcode::Uextend, 64) => NarrowValueMode::ZeroExtend64,
-                (Opcode::Sextend, 32) => NarrowValueMode::SignExtend32,
-                (Opcode::Sextend, 64) => NarrowValueMode::SignExtend64,
-                _ => unreachable!(),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::gen_move(rd, rn, ty));
+        Opcode::GlobalValue => {
+            panic!("global_value should have been removed by legalization!");
         }
 
-        Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => {
-            let ty = ty.unwrap();
-            let size = ty_bits(ty);
-            let narrow_mode = match (op, size) {
-                (Opcode::Ishl, _) => NarrowValueMode::None,
-                (Opcode::Ushr, 64) => NarrowValueMode::ZeroExtend64,
-                (Opcode::Ushr, _) => NarrowValueMode::ZeroExtend32,
-                (Opcode::Sshr, 64) => NarrowValueMode::SignExtend64,
-                (Opcode::Sshr, _) => NarrowValueMode::SignExtend32,
-                _ => unreachable!(),
-            };
-            let shift_op = match op {
-                Opcode::Ishl => choose_32_64(ty, ShiftOp::LShL32, ShiftOp::LShL64),
-                Opcode::Ushr => choose_32_64(ty, ShiftOp::LShR32, ShiftOp::LShR64),
-                Opcode::Sshr => choose_32_64(ty, ShiftOp::AShR32, ShiftOp::AShR64),
-                _ => unreachable!(),
-            };
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-            if let Some(imm) = input_matches_const(ctx, inputs[1]) {
-                let shift_imm = (imm & (size as u64 - 1)) as u8;
-                let shift_reg = zero_reg();
-                ctx.emit(Inst::ShiftRR {
-                    shift_op,
-                    rd,
-                    rn,
-                    shift_imm,
-                    shift_reg,
-                });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                let shift_imm = 0;
-                let shift_reg = if size < 64 {
-                    let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                    ctx.emit(Inst::gen_move(tmp, rm, types::I64));
-                    ctx.emit(Inst::AluRUImm16Shifted {
-                        alu_op: ALUOp::And64,
-                        rd: tmp,
-                        imm: UImm16Shifted::maybe_from_u64(size as u64 - 1).unwrap(),
-                    });
-                    tmp.to_reg()
-                } else {
-                    rm
-                };
-                ctx.emit(Inst::ShiftRR {
-                    shift_op,
-                    rd,
-                    rn,
-                    shift_imm,
-                    shift_reg,
-                });
-            }
+        Opcode::TlsValue => {
+            unimplemented!("Thread-local storage support not implemented!");
         }
 
-        Opcode::Rotr | Opcode::Rotl => {
-            // s390x doesn't have a right-rotate instruction, but a right rotation of K places is
-            // effectively a left rotation of N - K places, if N is the integer's bit size. We
-            // implement right rotations with this trick.
-            //
-            // For a 32-bit or 64-bit rotate-left, we can use the ROR instruction directly.
-            //
-            // For a < 32-bit rotate-left, we synthesize this as:
-            //
-            //    rotr rd, rn, rm
-            //
-            //       =>
-            //
-            //    zero-extend rn, <32-or-64>
-            //    and tmp_masked_rm, rm, <bitwidth - 1>
-            //    sub tmp1, tmp_masked_rm, <bitwidth>
-            //    sub tmp1, zero, tmp1  ; neg
-            //    lsr tmp2, rn, tmp_masked_rm
-            //    lsl rd, rn, tmp1
-            //    orr rd, rd, tmp2
-            //
-            // For a constant amount, we can instead do:
-            //
-            //    zero-extend rn, <32-or-64>
-            //    lsr tmp2, rn, #<shiftimm>
-            //    lsl rd, rn, <bitwidth - shiftimm>
-            //    orr rd, rd, tmp2
-
-            let is_rotr = op == Opcode::Rotr;
-
-            let ty = ty.unwrap();
-            let ty_bits_size = ty_bits(ty) as u64;
-
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(
-                ctx,
-                inputs[0],
-                if ty_bits_size <= 32 {
-                    NarrowValueMode::ZeroExtend32
-                } else {
-                    NarrowValueMode::ZeroExtend64
-                },
-            );
-
-            if ty_bits_size == 32 || ty_bits_size == 64 {
-                let shift_op = choose_32_64(ty, ShiftOp::RotL32, ShiftOp::RotL64);
-                if let Some(imm) = input_matches_const(ctx, inputs[1]) {
-                    let shiftcount = imm & (ty_bits_size - 1);
-                    let shiftcount = if is_rotr {
-                        ty_bits_size - shiftcount
-                    } else {
-                        shiftcount
-                    };
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op,
-                        rd,
-                        rn,
-                        shift_imm: shiftcount as u8,
-                        shift_reg: zero_reg(),
-                    });
-                } else {
-                    let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                    let rm = if is_rotr {
-                        // Really ty_bits_size - rn, but the upper bits of the result are
-                        // ignored (because of the implicit masking done by the instruction),
-                        // so this is equivalent to negating the input.
-                        let op = choose_32_64(ty, UnaryOp::Neg32, UnaryOp::Neg64);
-                        let tmp = ctx.alloc_tmp(ty).only_reg().unwrap();
-                        ctx.emit(Inst::UnaryRR {
-                            op,
-                            rd: tmp,
-                            rn: rm,
-                        });
-                        tmp.to_reg()
-                    } else {
-                        rm
-                    };
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op,
-                        rd,
-                        rn,
-                        shift_imm: 0,
-                        shift_reg: rm,
-                    });
-                }
-            } else {
-                debug_assert!(ty_bits_size < 32);
-
-                if let Some(imm) = input_matches_const(ctx, inputs[1]) {
-                    let rot_count = imm & (ty_bits_size - 1);
-                    let (lshl_count, lshr_count) = if is_rotr {
-                        (ty_bits_size - rot_count, rot_count)
-                    } else {
-                        (rot_count, ty_bits_size - rot_count)
-                    };
+        Opcode::GetPinnedReg | Opcode::SetPinnedReg => {
+            unimplemented!("Pinned register support not implemented!");
+        }
 
-                    let tmp1 = ctx.alloc_tmp(types::I32).only_reg().unwrap();
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op: ShiftOp::LShL32,
-                        rd: tmp1,
-                        rn,
-                        shift_imm: lshl_count as u8,
-                        shift_reg: zero_reg(),
-                    });
-
-                    let tmp2 = ctx.alloc_tmp(types::I32).only_reg().unwrap();
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op: ShiftOp::LShR32,
-                        rd: tmp2,
-                        rn,
-                        shift_imm: lshr_count as u8,
-                        shift_reg: zero_reg(),
-                    });
-
-                    ctx.emit(Inst::AluRRR {
-                        alu_op: ALUOp::Orr32,
-                        rd,
-                        rn: tmp1.to_reg(),
-                        rm: tmp2.to_reg(),
-                    });
-                } else {
-                    let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                    let tmp1 = ctx.alloc_tmp(types::I32).only_reg().unwrap();
-                    let tmp2 = ctx.alloc_tmp(types::I32).only_reg().unwrap();
-
-                    ctx.emit(Inst::mov32(tmp1, rm));
-                    ctx.emit(Inst::UnaryRR {
-                        op: UnaryOp::Neg32,
-                        rd: tmp2,
-                        rn: rm,
-                    });
-
-                    ctx.emit(Inst::AluRUImm16Shifted {
-                        alu_op: ALUOp::And32,
-                        rd: tmp1,
-                        imm: UImm16Shifted::maybe_from_u64(ty_bits_size - 1).unwrap(),
-                    });
-                    ctx.emit(Inst::AluRUImm16Shifted {
-                        alu_op: ALUOp::And32,
-                        rd: tmp2,
-                        imm: UImm16Shifted::maybe_from_u64(ty_bits_size - 1).unwrap(),
-                    });
-
-                    let (lshr, lshl) = if is_rotr { (tmp2, tmp1) } else { (tmp1, tmp2) };
-
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op: ShiftOp::LShL32,
-                        rd: lshl,
-                        rn,
-                        shift_imm: 0,
-                        shift_reg: lshl.to_reg(),
-                    });
-
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op: ShiftOp::LShR32,
-                        rd: lshr,
-                        rn,
-                        shift_imm: 0,
-                        shift_reg: lshr.to_reg(),
-                    });
-
-                    ctx.emit(Inst::AluRRR {
-                        alu_op: ALUOp::Orr32,
-                        rd,
-                        rn: lshl.to_reg(),
-                        rm: lshr.to_reg(),
-                    });
-                }
-            }
+        Opcode::Trap | Opcode::ResumableTrap => {
+            let trap_code = ctx.data(insn).trap_code().unwrap();
+            ctx.emit_safepoint(Inst::Trap { trap_code })
         }
 
-        Opcode::Bnot => {
-            let ty = ty.unwrap();
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if isa_flags.has_mie2() {
-                ctx.emit(Inst::AluRRR {
-                    alu_op: choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64),
-                    rd,
-                    rn,
-                    rm: rn,
-                });
-            } else {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                lower_bnot(ctx, ty, rd);
-            }
-        }
-
-        Opcode::Band => {
-            let ty = ty.unwrap();
-            let alu_op = choose_32_64(ty, ALUOp::And32, ALUOp::And64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if let Some(imm) = input_matches_uimm16shifted_inv(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm16Shifted { alu_op, rd, imm });
-            } else if let Some(imm) = input_matches_uimm32shifted_inv(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm32Shifted { alu_op, rd, imm });
-            } else if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            }
-        }
-
-        Opcode::Bor => {
-            let ty = ty.unwrap();
-            let alu_op = choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if let Some(imm) = input_matches_uimm16shifted(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm16Shifted { alu_op, rd, imm });
-            } else if let Some(imm) = input_matches_uimm32shifted(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm32Shifted { alu_op, rd, imm });
-            } else if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            }
-        }
-
-        Opcode::Bxor => {
-            let ty = ty.unwrap();
-            let alu_op = choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if let Some(imm) = input_matches_uimm32shifted(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm32Shifted { alu_op, rd, imm });
-            } else if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRX { alu_op, rd, mem });
-            } else {
-                let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-                ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            }
-        }
-
-        Opcode::BandNot | Opcode::BorNot | Opcode::BxorNot => {
-            let ty = ty.unwrap();
-            let alu_op = match (op, isa_flags.has_mie2()) {
-                (Opcode::BandNot, true) => choose_32_64(ty, ALUOp::AndNot32, ALUOp::AndNot64),
-                (Opcode::BorNot, true) => choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64),
-                (Opcode::BxorNot, true) => choose_32_64(ty, ALUOp::XorNot32, ALUOp::XorNot64),
-                (Opcode::BandNot, false) => choose_32_64(ty, ALUOp::And32, ALUOp::And64),
-                (Opcode::BorNot, false) => choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64),
-                (Opcode::BxorNot, false) => choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64),
-                _ => unreachable!(),
-            };
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
-            if !isa_flags.has_mie2() {
-                lower_bnot(ctx, ty, rd);
-            }
-        }
-
-        Opcode::Bitselect => {
-            let ty = ty.unwrap();
-            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rcond = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-            ctx.emit(Inst::AluRRR {
-                alu_op: choose_32_64(ty, ALUOp::And32, ALUOp::And64),
-                rd: tmp,
-                rn,
-                rm: rcond,
-            });
-            if isa_flags.has_mie2() {
-                ctx.emit(Inst::AluRRR {
-                    alu_op: choose_32_64(ty, ALUOp::AndNot32, ALUOp::AndNot64),
-                    rd,
-                    rn: rm,
-                    rm: rcond,
-                });
-            } else {
-                ctx.emit(Inst::AluRRR {
-                    alu_op: choose_32_64(ty, ALUOp::And32, ALUOp::And64),
-                    rd,
-                    rn: rm,
-                    rm: rcond,
-                });
-                lower_bnot(ctx, ty, rd);
-            }
-            ctx.emit(Inst::AluRRR {
-                alu_op: choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64),
-                rd,
-                rn: rd.to_reg(),
-                rm: tmp.to_reg(),
-            });
-        }
-
-        Opcode::Bextend | Opcode::Bmask => {
-            // Bextend and Bmask both simply sign-extend. This works for:
-            // - Bextend, because booleans are stored as 0 / -1, so we
-            //   sign-extend the -1 to a -1 in the wider width.
-            // - Bmask, because the resulting integer mask value must be
-            //   all-ones (-1) if the argument is true.
-            //
-            // For a sign-extension from a 1-bit value (Case 1 below), we need
-            // to do things a bit specially, because the ISA does not have a
-            // 1-to-N-bit sign extension instruction.  For 8-bit or wider
-            // sources (Case 2 below), we do a sign extension normally.
-
-            let from_ty = ctx.input_ty(insn, 0);
-            let to_ty = ctx.output_ty(insn, 0);
-            let from_bits = ty_bits(from_ty);
-            let to_bits = ty_bits(to_ty);
-
-            assert!(
-                from_bits <= 64 && to_bits <= 64,
-                "Vector Bextend not supported yet"
-            );
-
-            if from_bits >= to_bits {
-                // Just a move.
-                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                let ty = ctx.input_ty(insn, 0);
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-            } else if from_bits == 1 {
-                assert!(to_bits >= 8);
-                // Case 1: 1-bit to N-bit extension: use a shift-left /
-                // shift-right sequence to create a 0 / -1 result.
-                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                let shl_op = choose_32_64(to_ty, ShiftOp::LShL32, ShiftOp::LShL64);
-                let shr_op = choose_32_64(to_ty, ShiftOp::AShR32, ShiftOp::AShR64);
-                let count = if to_bits > 32 { 63 } else { 31 };
-                ctx.emit(Inst::ShiftRR {
-                    shift_op: shl_op,
-                    rd,
-                    rn,
-                    shift_imm: count,
-                    shift_reg: zero_reg(),
-                });
-                ctx.emit(Inst::ShiftRR {
-                    shift_op: shr_op,
-                    rd,
-                    rn: rd.to_reg(),
-                    shift_imm: count,
-                    shift_reg: zero_reg(),
-                });
-            } else {
-                // Case 2: 8-or-more-bit to N-bit extension: just sign-extend. A
-                // `true` (all ones, or `-1`) will be extended to -1 with the
-                // larger width.
-                assert!(from_bits >= 8);
-                let narrow_mode = if to_bits == 64 {
-                    NarrowValueMode::SignExtend64
-                } else {
-                    assert!(to_bits <= 32);
-                    NarrowValueMode::SignExtend32
-                };
-                let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-                let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-                ctx.emit(Inst::gen_move(rd, rn, to_ty));
-            }
-        }
-
-        Opcode::Bint => {
-            // Booleans are stored as all-zeroes (0) or all-ones (-1). We AND
-            // out the LSB to give a 0 / 1-valued integer result.
-            let ty = ty.unwrap();
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            if ty_bits(ty) <= 16 {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm16Shifted {
-                    alu_op: ALUOp::And32,
-                    rd,
-                    imm: UImm16Shifted::maybe_from_u64(1).unwrap(),
-                });
-            } else if ty_bits(ty) <= 32 {
-                ctx.emit(Inst::gen_move(rd, rn, ty));
-                ctx.emit(Inst::AluRUImm32Shifted {
-                    alu_op: ALUOp::And32,
-                    rd,
-                    imm: UImm32Shifted::maybe_from_u64(1).unwrap(),
-                });
-            } else {
-                let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                lower_constant_u64(ctx, tmp, 1);
-                ctx.emit(Inst::AluRRR {
-                    alu_op: ALUOp::And64,
-                    rd,
-                    rn,
-                    rm: tmp.to_reg(),
-                });
-            }
-        }
-
-        Opcode::Clz => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let ty = ty.unwrap();
-            let ty_bits_size = ty_bits(ty);
-
-            let rn = if ty_bits_size < 64 {
-                let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                ctx.emit(Inst::Extend {
-                    rd: tmp,
-                    rn,
-                    signed: false,
-                    from_bits: ty_bits_size as u8,
-                    to_bits: 64,
-                });
-                tmp.to_reg()
-            } else {
-                rn
-            };
-
-            ctx.emit(Inst::Flogr { rn });
-            ctx.emit(Inst::gen_move(rd, gpr(0), ty));
-
-            if ty_bits_size < 64 {
-                ctx.emit(Inst::AluRSImm16 {
-                    alu_op: ALUOp::Add32,
-                    rd,
-                    imm: -(64 - ty_bits_size as i16),
-                });
-            }
-        }
-
-        Opcode::Cls => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let ty = ty.unwrap();
-            let ty_bits_size = ty_bits(ty);
-
-            let rn = if ty_bits_size < 64 {
-                let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                ctx.emit(Inst::Extend {
-                    rd: tmp,
-                    rn,
-                    signed: true,
-                    from_bits: ty_bits_size as u8,
-                    to_bits: 64,
-                });
-                tmp.to_reg()
-            } else {
-                rn
-            };
-
-            // tmp = rn ^ ((signed)rn >> 63)
-            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-            ctx.emit(Inst::ShiftRR {
-                shift_op: ShiftOp::AShR64,
-                rd: tmp,
-                rn,
-                shift_imm: 63,
-                shift_reg: zero_reg(),
-            });
-            ctx.emit(Inst::AluRRR {
-                alu_op: ALUOp::Xor64,
-                rd: tmp,
-                rn: tmp.to_reg(),
-                rm: rn,
-            });
-
-            ctx.emit(Inst::Flogr { rn: tmp.to_reg() });
-            ctx.emit(Inst::gen_move(rd, gpr(0), ty));
-
-            if ty_bits_size < 64 {
-                ctx.emit(Inst::AluRSImm16 {
-                    alu_op: ALUOp::Add32,
-                    rd,
-                    imm: -(64 - ty_bits_size as i16),
-                });
-            }
-        }
-
-        Opcode::Ctz => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let ty = ty.unwrap();
-            let ty_bits_size = ty_bits(ty);
-
-            let rn = if ty_bits_size < 64 {
-                let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                ctx.emit(Inst::gen_move(tmp, rn, ty));
-                ctx.emit(Inst::AluRUImm16Shifted {
-                    alu_op: ALUOp::Orr64,
-                    rd: tmp,
-                    imm: UImm16Shifted::maybe_from_u64(1u64 << ty_bits_size).unwrap(),
-                });
-                tmp.to_reg()
-            } else {
-                rn
-            };
-
-            // tmp = rn & -rn
-            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-            ctx.emit(Inst::UnaryRR {
-                op: UnaryOp::Neg64,
-                rd: tmp,
-                rn,
-            });
-            ctx.emit(Inst::AluRRR {
-                alu_op: ALUOp::And64,
-                rd: tmp,
-                rn: tmp.to_reg(),
-                rm: rn,
-            });
-
-            ctx.emit(Inst::Flogr { rn: tmp.to_reg() });
-            if ty_bits_size == 64 {
-                ctx.emit(Inst::CMov64SImm16 {
-                    rd: writable_gpr(0),
-                    cond: Cond::from_intcc(IntCC::Equal),
-                    imm: -1,
-                });
-            }
-
-            if ty_bits_size <= 32 {
-                lower_constant_u32(ctx, rd, 63);
-            } else {
-                lower_constant_u64(ctx, rd, 63);
-            }
-            let alu_op = choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64);
-            ctx.emit(Inst::AluRRR {
-                alu_op,
-                rd,
-                rn: rd.to_reg(),
-                rm: gpr(0),
-            });
-        }
-
-        Opcode::Bitrev => unimplemented!(),
-
-        Opcode::Popcnt => {
-            let ty = ty.unwrap();
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            if ty_bits(ty) <= 8 {
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                ctx.emit(Inst::UnaryRR {
-                    op: UnaryOp::PopcntByte,
-                    rd,
-                    rn,
-                });
-            } else if isa_flags.has_mie2() {
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64);
-                ctx.emit(Inst::UnaryRR {
-                    op: UnaryOp::PopcntReg,
-                    rd,
-                    rn,
-                });
-            } else {
-                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                ctx.emit(Inst::UnaryRR {
-                    op: UnaryOp::PopcntByte,
-                    rd,
-                    rn,
-                });
-                let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                let mut shift = ty_bits(ty) as u8;
-                while shift > 8 {
-                    shift = shift / 2;
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op: choose_32_64(ty, ShiftOp::LShL32, ShiftOp::LShL64),
-                        rd: tmp,
-                        rn: rd.to_reg(),
-                        shift_imm: shift,
-                        shift_reg: zero_reg(),
-                    });
-                    ctx.emit(Inst::AluRR {
-                        alu_op: choose_32_64(ty, ALUOp::Add32, ALUOp::Add64),
-                        rd,
-                        rm: tmp.to_reg(),
-                    });
-                }
-                let shift = ty_bits(ty) as u8 - 8;
-                ctx.emit(Inst::ShiftRR {
-                    shift_op: choose_32_64(ty, ShiftOp::LShR32, ShiftOp::LShR64),
-                    rd,
-                    rn: rd.to_reg(),
-                    shift_imm: shift,
-                    shift_reg: zero_reg(),
-                });
-            }
-        }
-
-        Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => {
-            let bits = ty_bits(ctx.output_ty(insn, 0));
-            let fpu_op = match (op, bits) {
-                (Opcode::Fadd, 32) => FPUOp2::Add32,
-                (Opcode::Fadd, 64) => FPUOp2::Add64,
-                (Opcode::Fsub, 32) => FPUOp2::Sub32,
-                (Opcode::Fsub, 64) => FPUOp2::Sub64,
-                (Opcode::Fmul, 32) => FPUOp2::Mul32,
-                (Opcode::Fmul, 64) => FPUOp2::Mul64,
-                (Opcode::Fdiv, 32) => FPUOp2::Div32,
-                (Opcode::Fdiv, 64) => FPUOp2::Div64,
-                _ => panic!("Unknown op/bits combination"),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::mov64(rd, rn));
-            ctx.emit(Inst::FpuRRR { fpu_op, rd, rm });
-        }
-
-        Opcode::Fmin | Opcode::Fmax => {
-            let bits = ty_bits(ctx.output_ty(insn, 0));
-            let fpu_op = match (op, bits) {
-                (Opcode::Fmin, 32) => FPUOp2::Min32,
-                (Opcode::Fmin, 64) => FPUOp2::Min64,
-                (Opcode::Fmax, 32) => FPUOp2::Max32,
-                (Opcode::Fmax, 64) => FPUOp2::Max64,
-                _ => panic!("Unknown op/bits combination"),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::FpuVecRRR { fpu_op, rd, rn, rm });
-        }
-
-        Opcode::Sqrt | Opcode::Fneg | Opcode::Fabs | Opcode::Fpromote | Opcode::Fdemote => {
-            let bits = ty_bits(ctx.output_ty(insn, 0));
-            let fpu_op = match (op, bits) {
-                (Opcode::Sqrt, 32) => FPUOp1::Sqrt32,
-                (Opcode::Sqrt, 64) => FPUOp1::Sqrt64,
-                (Opcode::Fneg, 32) => FPUOp1::Neg32,
-                (Opcode::Fneg, 64) => FPUOp1::Neg64,
-                (Opcode::Fabs, 32) => FPUOp1::Abs32,
-                (Opcode::Fabs, 64) => FPUOp1::Abs64,
-                (Opcode::Fpromote, 32) => panic!("Cannot promote to 32 bits"),
-                (Opcode::Fpromote, 64) => FPUOp1::Cvt32To64,
-                (Opcode::Fdemote, 32) => FPUOp1::Cvt64To32,
-                (Opcode::Fdemote, 64) => panic!("Cannot demote to 64 bits"),
-                _ => panic!("Unknown op/bits combination"),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::FpuRR { fpu_op, rd, rn });
-        }
-
-        Opcode::Ceil | Opcode::Floor | Opcode::Trunc | Opcode::Nearest => {
-            let bits = ty_bits(ctx.output_ty(insn, 0));
-            let op = match (op, bits) {
-                (Opcode::Ceil, 32) => FpuRoundMode::Plus32,
-                (Opcode::Ceil, 64) => FpuRoundMode::Plus64,
-                (Opcode::Floor, 32) => FpuRoundMode::Minus32,
-                (Opcode::Floor, 64) => FpuRoundMode::Minus64,
-                (Opcode::Trunc, 32) => FpuRoundMode::Zero32,
-                (Opcode::Trunc, 64) => FpuRoundMode::Zero64,
-                (Opcode::Nearest, 32) => FpuRoundMode::Nearest32,
-                (Opcode::Nearest, 64) => FpuRoundMode::Nearest64,
-                _ => panic!("Unknown op/bits combination"),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::FpuRound { op, rd, rn });
-        }
-
-        Opcode::Fma => {
-            let bits = ty_bits(ctx.output_ty(insn, 0));
-            let fpu_op = match bits {
-                32 => FPUOp3::MAdd32,
-                64 => FPUOp3::MAdd64,
-                _ => panic!("Unknown op size"),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let ra = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::mov64(rd, ra));
-            ctx.emit(Inst::FpuRRRR { fpu_op, rd, rn, rm });
-        }
-
-        Opcode::Fcopysign => {
-            let ty = ctx.output_ty(insn, 0);
-            let bits = ty_bits(ty) as u8;
-            assert!(bits == 32 || bits == 64);
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-            ctx.emit(Inst::FpuCopysign { rd, rn, rm });
-        }
-
-        Opcode::FcvtFromUint | Opcode::FcvtFromSint => {
-            let in_bits = ty_bits(ctx.input_ty(insn, 0));
-            let out_bits = ty_bits(ctx.output_ty(insn, 0));
-            let signed = op == Opcode::FcvtFromSint;
-            let op = match (signed, in_bits, out_bits) {
-                (false, 32, 32) => IntToFpuOp::U32ToF32,
-                (true, 32, 32) => IntToFpuOp::I32ToF32,
-                (false, 32, 64) => IntToFpuOp::U32ToF64,
-                (true, 32, 64) => IntToFpuOp::I32ToF64,
-                (false, 64, 32) => IntToFpuOp::U64ToF32,
-                (true, 64, 32) => IntToFpuOp::I64ToF32,
-                (false, 64, 64) => IntToFpuOp::U64ToF64,
-                (true, 64, 64) => IntToFpuOp::I64ToF64,
-                _ => panic!("Unknown input/output-bits combination"),
-            };
-            let narrow_mode = match (signed, in_bits) {
-                (false, 32) => NarrowValueMode::ZeroExtend32,
-                (true, 32) => NarrowValueMode::SignExtend32,
-                (false, 64) => NarrowValueMode::ZeroExtend64,
-                (true, 64) => NarrowValueMode::SignExtend64,
-                _ => panic!("Unknown input size"),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            ctx.emit(Inst::IntToFpu { op, rd, rn });
-        }
-
-        Opcode::FcvtToUint | Opcode::FcvtToSint => {
-            let in_bits = ty_bits(ctx.input_ty(insn, 0));
-            let out_bits = ty_bits(ctx.output_ty(insn, 0));
-            let signed = op == Opcode::FcvtToSint;
-            let op = match (signed, in_bits, out_bits) {
-                (false, 32, 32) => FpuToIntOp::F32ToU32,
-                (true, 32, 32) => FpuToIntOp::F32ToI32,
-                (false, 32, 64) => FpuToIntOp::F32ToU64,
-                (true, 32, 64) => FpuToIntOp::F32ToI64,
-                (false, 64, 32) => FpuToIntOp::F64ToU32,
-                (true, 64, 32) => FpuToIntOp::F64ToI32,
-                (false, 64, 64) => FpuToIntOp::F64ToU64,
-                (true, 64, 64) => FpuToIntOp::F64ToI64,
-                _ => panic!("Unknown input/output-bits combination"),
-            };
-
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-            // First, check whether the input is a NaN and trap if so.
-            if in_bits == 32 {
-                ctx.emit(Inst::FpuCmp32 { rn, rm: rn });
-            } else {
-                ctx.emit(Inst::FpuCmp64 { rn, rm: rn });
-            }
-            ctx.emit(Inst::TrapIf {
-                trap_code: TrapCode::BadConversionToInteger,
-                cond: Cond::from_floatcc(FloatCC::Unordered),
-            });
-
-            // Perform the conversion.  If this sets CC 3, we have a
-            // "special case".  Since we already exluded the case where
-            // the input was a NaN, the only other option is that the
-            // conversion overflowed the target type.
-            ctx.emit(Inst::FpuToInt { op, rd, rn });
-            ctx.emit(Inst::TrapIf {
-                trap_code: TrapCode::IntegerOverflow,
-                cond: Cond::from_floatcc(FloatCC::Unordered),
-            });
-        }
-
-        Opcode::FcvtToUintSat | Opcode::FcvtToSintSat => {
-            let in_bits = ty_bits(ctx.input_ty(insn, 0));
-            let out_bits = ty_bits(ctx.output_ty(insn, 0));
-            let signed = op == Opcode::FcvtToSintSat;
-            let op = match (signed, in_bits, out_bits) {
-                (false, 32, 32) => FpuToIntOp::F32ToU32,
-                (true, 32, 32) => FpuToIntOp::F32ToI32,
-                (false, 32, 64) => FpuToIntOp::F32ToU64,
-                (true, 32, 64) => FpuToIntOp::F32ToI64,
-                (false, 64, 32) => FpuToIntOp::F64ToU32,
-                (true, 64, 32) => FpuToIntOp::F64ToI32,
-                (false, 64, 64) => FpuToIntOp::F64ToU64,
-                (true, 64, 64) => FpuToIntOp::F64ToI64,
-                _ => panic!("Unknown input/output-bits combination"),
-            };
-
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-            // Perform the conversion.
-            ctx.emit(Inst::FpuToInt { op, rd, rn });
-
-            // In most special cases, the Z instruction already yields the
-            // result expected by Cranelift semantic.  The only exception
-            // it the case where the input was a Nan.  We explicitly check
-            // for that and force the output to 0 in that case.
-            if in_bits == 32 {
-                ctx.emit(Inst::FpuCmp32 { rn, rm: rn });
-            } else {
-                ctx.emit(Inst::FpuCmp64 { rn, rm: rn });
-            }
-            let cond = Cond::from_floatcc(FloatCC::Unordered);
-            if out_bits <= 32 {
-                ctx.emit(Inst::CMov32SImm16 { rd, cond, imm: 0 });
-            } else {
-                ctx.emit(Inst::CMov64SImm16 { rd, cond, imm: 0 });
-            }
-        }
-
-        Opcode::FcvtLowFromSint => unimplemented!("FcvtLowFromSint"),
-
-        Opcode::Bitcast => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let input_ty = ctx.input_ty(insn, 0);
-            let output_ty = ctx.output_ty(insn, 0);
-            lower_bitcast(ctx, rd, output_ty, rn, input_ty);
-        }
-
-        Opcode::Load
-        | Opcode::Uload8
-        | Opcode::Sload8
-        | Opcode::Uload16
-        | Opcode::Sload16
-        | Opcode::Uload32
-        | Opcode::Sload32
-        | Opcode::LoadComplex
-        | Opcode::Uload8Complex
-        | Opcode::Sload8Complex
-        | Opcode::Uload16Complex
-        | Opcode::Sload16Complex
-        | Opcode::Uload32Complex
-        | Opcode::Sload32Complex => {
-            let off = ctx.data(insn).load_store_offset().unwrap();
-            let flags = ctx.memflags(insn).unwrap();
-            let endianness = flags.endianness(Endianness::Big);
-            let elem_ty = ctx.output_ty(insn, 0);
-            let is_float = ty_is_float(elem_ty);
-            let to_bits = ty_bits(elem_ty);
-            let from_bits = match op {
-                Opcode::Load | Opcode::LoadComplex => to_bits,
-                Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => {
-                    8
-                }
-                Opcode::Sload16
-                | Opcode::Uload16
-                | Opcode::Sload16Complex
-                | Opcode::Uload16Complex => 16,
-                Opcode::Sload32
-                | Opcode::Uload32
-                | Opcode::Sload32Complex
-                | Opcode::Uload32Complex => 32,
-                _ => unreachable!(),
-            };
-            let ext_bits = if to_bits < 32 { 32 } else { to_bits };
-            let sign_extend = match op {
-                Opcode::Sload8
-                | Opcode::Sload8Complex
-                | Opcode::Sload16
-                | Opcode::Sload16Complex
-                | Opcode::Sload32
-                | Opcode::Sload32Complex => true,
-                _ => false,
-            };
-
-            let mem = lower_address(ctx, &inputs[..], off, flags);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-            if endianness == Endianness::Big {
-                ctx.emit(match (ext_bits, from_bits, sign_extend, is_float) {
-                    (32, 32, _, true) => Inst::FpuLoad32 { rd, mem },
-                    (64, 64, _, true) => Inst::FpuLoad64 { rd, mem },
-                    (32, 32, _, false) => Inst::Load32 { rd, mem },
-                    (64, 64, _, false) => Inst::Load64 { rd, mem },
-                    (32, 8, false, _) => Inst::Load32ZExt8 { rd, mem },
-                    (32, 8, true, _) => Inst::Load32SExt8 { rd, mem },
-                    (32, 16, false, _) => Inst::Load32ZExt16 { rd, mem },
-                    (32, 16, true, _) => Inst::Load32SExt16 { rd, mem },
-                    (64, 8, false, _) => Inst::Load64ZExt8 { rd, mem },
-                    (64, 8, true, _) => Inst::Load64SExt8 { rd, mem },
-                    (64, 16, false, _) => Inst::Load64ZExt16 { rd, mem },
-                    (64, 16, true, _) => Inst::Load64SExt16 { rd, mem },
-                    (64, 32, false, _) => Inst::Load64ZExt32 { rd, mem },
-                    (64, 32, true, _) => Inst::Load64SExt32 { rd, mem },
-                    _ => panic!("Unsupported size in load"),
-                });
-            } else if !is_float {
-                ctx.emit(match (ext_bits, from_bits, sign_extend) {
-                    (_, 16, _) => Inst::LoadRev16 { rd, mem },
-                    (_, 32, _) => Inst::LoadRev32 { rd, mem },
-                    (_, 64, _) => Inst::LoadRev64 { rd, mem },
-                    (32, 8, false) => Inst::Load32ZExt8 { rd, mem },
-                    (32, 8, true) => Inst::Load32SExt8 { rd, mem },
-                    (64, 8, false) => Inst::Load64ZExt8 { rd, mem },
-                    (64, 8, true) => Inst::Load64SExt8 { rd, mem },
-                    _ => panic!("Unsupported size in load"),
-                });
-                if to_bits > from_bits && from_bits > 8 {
-                    ctx.emit(Inst::Extend {
-                        rd,
-                        rn: rd.to_reg(),
-                        signed: sign_extend,
-                        from_bits: from_bits as u8,
-                        to_bits: to_bits as u8,
-                    });
-                }
-            } else if isa_flags.has_vxrs_ext2() {
-                ctx.emit(match from_bits {
-                    32 => Inst::FpuLoadRev32 { rd, mem },
-                    64 => Inst::FpuLoadRev64 { rd, mem },
-                    _ => panic!("Unsupported size in load"),
-                });
-            } else {
-                match from_bits {
-                    32 => {
-                        let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap();
-                        ctx.emit(Inst::LoadRev32 { rd: tmp, mem });
-                        lower_bitcast(ctx, rd, elem_ty, tmp.to_reg(), types::I32);
-                    }
-                    64 => {
-                        let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                        ctx.emit(Inst::LoadRev64 { rd: tmp, mem });
-                        lower_bitcast(ctx, rd, elem_ty, tmp.to_reg(), types::I64);
-                    }
-                    _ => panic!("Unsupported size in load"),
-                }
-            }
-        }
-
-        Opcode::Store
-        | Opcode::Istore8
-        | Opcode::Istore16
-        | Opcode::Istore32
-        | Opcode::StoreComplex
-        | Opcode::Istore8Complex
-        | Opcode::Istore16Complex
-        | Opcode::Istore32Complex => {
-            let off = ctx.data(insn).load_store_offset().unwrap();
-            let flags = ctx.memflags(insn).unwrap();
-            let endianness = flags.endianness(Endianness::Big);
-            let elem_ty = match op {
-                Opcode::Istore8 | Opcode::Istore8Complex => types::I8,
-                Opcode::Istore16 | Opcode::Istore16Complex => types::I16,
-                Opcode::Istore32 | Opcode::Istore32Complex => types::I32,
-                Opcode::Store | Opcode::StoreComplex => ctx.input_ty(insn, 0),
-                _ => unreachable!(),
-            };
-
-            let mem = lower_address(ctx, &inputs[1..], off, flags);
-
-            if ty_is_float(elem_ty) {
-                let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                if endianness == Endianness::Big {
-                    ctx.emit(match ty_bits(elem_ty) {
-                        32 => Inst::FpuStore32 { rd, mem },
-                        64 => Inst::FpuStore64 { rd, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                } else if isa_flags.has_vxrs_ext2() {
-                    ctx.emit(match ty_bits(elem_ty) {
-                        32 => Inst::FpuStoreRev32 { rd, mem },
-                        64 => Inst::FpuStoreRev64 { rd, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                } else {
-                    match ty_bits(elem_ty) {
-                        32 => {
-                            let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap();
-                            lower_bitcast(ctx, tmp, types::I32, rd, elem_ty);
-                            ctx.emit(Inst::StoreRev32 {
-                                rd: tmp.to_reg(),
-                                mem,
-                            });
-                        }
-                        64 => {
-                            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                            lower_bitcast(ctx, tmp, types::I64, rd, elem_ty);
-                            ctx.emit(Inst::StoreRev64 {
-                                rd: tmp.to_reg(),
-                                mem,
-                            });
-                        }
-                        _ => panic!("Unsupported size in load"),
-                    }
-                }
-            } else if ty_bits(elem_ty) <= 16 {
-                if let Some(imm) = input_matches_const(ctx, inputs[0]) {
-                    ctx.emit(match (endianness, ty_bits(elem_ty)) {
-                        (_, 1) | (_, 8) => Inst::StoreImm8 {
-                            imm: imm as u8,
-                            mem,
-                        },
-                        (Endianness::Big, 16) => Inst::StoreImm16 {
-                            imm: imm as i16,
-                            mem,
-                        },
-                        (Endianness::Little, 16) => Inst::StoreImm16 {
-                            imm: (imm as i16).swap_bytes(),
-                            mem,
-                        },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                } else {
-                    let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                    ctx.emit(match (endianness, ty_bits(elem_ty)) {
-                        (_, 1) | (_, 8) => Inst::Store8 { rd, mem },
-                        (Endianness::Big, 16) => Inst::Store16 { rd, mem },
-                        (Endianness::Little, 16) => Inst::StoreRev16 { rd, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                }
-            } else if endianness == Endianness::Big {
-                if let Some(imm) = input_matches_simm16(ctx, inputs[0]) {
-                    ctx.emit(match ty_bits(elem_ty) {
-                        32 => Inst::StoreImm32SExt16 { imm, mem },
-                        64 => Inst::StoreImm64SExt16 { imm, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                } else {
-                    let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                    ctx.emit(match ty_bits(elem_ty) {
-                        32 => Inst::Store32 { rd, mem },
-                        64 => Inst::Store64 { rd, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                }
-            } else {
-                let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                ctx.emit(match ty_bits(elem_ty) {
-                    32 => Inst::StoreRev32 { rd, mem },
-                    64 => Inst::StoreRev64 { rd, mem },
-                    _ => panic!("Unsupported size in store"),
-                });
-            }
-        }
-
-        Opcode::StackLoad | Opcode::StackStore => {
-            panic!("Direct stack memory access not supported; should not be used by Wasm");
-        }
-
-        Opcode::StackAddr => {
-            let (stack_slot, offset) = match *ctx.data(insn) {
-                InstructionData::StackLoad {
-                    opcode: Opcode::StackAddr,
-                    stack_slot,
-                    offset,
-                } => (stack_slot, offset),
-                _ => unreachable!(),
-            };
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let offset: i32 = offset.into();
-            let inst = ctx
-                .abi()
-                .stackslot_addr(stack_slot, u32::try_from(offset).unwrap(), rd);
-            ctx.emit(inst);
-        }
-
-        Opcode::ConstAddr => unimplemented!(),
-
-        Opcode::FuncAddr => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let (extname, dist) = ctx.call_target(insn).unwrap();
-            let extname = extname.clone();
-            if dist == RelocDistance::Near {
-                ctx.emit(Inst::LoadAddr {
-                    rd,
-                    mem: MemArg::Symbol {
-                        name: Box::new(extname),
-                        offset: 0,
-                        flags: MemFlags::trusted(),
-                    },
-                });
-            } else {
-                ctx.emit(Inst::LoadExtNameFar {
-                    rd,
-                    name: Box::new(extname),
-                    offset: 0,
-                });
-            }
-        }
-
-        Opcode::SymbolValue => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let (extname, dist, offset) = ctx.symbol_value(insn).unwrap();
-            let extname = extname.clone();
-            if dist == RelocDistance::Near && (offset & 1) == 0 && i32::try_from(offset).is_ok() {
-                ctx.emit(Inst::LoadAddr {
-                    rd,
-                    mem: MemArg::Symbol {
-                        name: Box::new(extname),
-                        offset: i32::try_from(offset).unwrap(),
-                        flags: MemFlags::trusted(),
-                    },
-                });
-            } else {
-                ctx.emit(Inst::LoadExtNameFar {
-                    rd,
-                    name: Box::new(extname),
-                    offset,
-                });
-            }
-        }
-
-        Opcode::HeapAddr => {
-            panic!("heap_addr should have been removed by legalization!");
-        }
-
-        Opcode::TableAddr => {
-            panic!("table_addr should have been removed by legalization!");
-        }
-
-        Opcode::GlobalValue => {
-            panic!("global_value should have been removed by legalization!");
-        }
-
-        Opcode::TlsValue => {
-            unimplemented!("Thread-local storage support not implemented!");
-        }
-
-        Opcode::GetPinnedReg | Opcode::SetPinnedReg => {
-            unimplemented!("Pinned register support not implemented!");
-        }
-
-        Opcode::Icmp => {
-            let condcode = ctx.data(insn).cond_code().unwrap();
-            let cond = Cond::from_intcc(condcode);
-            let is_signed = condcode_is_signed(condcode);
-            lower_icmp_to_flags(ctx, insn, is_signed, true);
-
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let ty = ctx.output_ty(insn, 0);
-            lower_flags_to_bool_result(ctx, cond, rd, ty);
-        }
-
-        Opcode::Fcmp => {
-            let condcode = ctx.data(insn).fp_cond_code().unwrap();
-            let cond = Cond::from_floatcc(condcode);
-            lower_fcmp_to_flags(ctx, insn);
-
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let ty = ctx.output_ty(insn, 0);
-            lower_flags_to_bool_result(ctx, cond, rd, ty);
-        }
-
-        Opcode::IsNull | Opcode::IsInvalid => {
-            // Null references are represented by the constant value 0; invalid
-            // references are represented by the constant value -1.
-            let cond = Cond::from_intcc(IntCC::Equal);
-            let imm = match op {
-                Opcode::IsNull => 0,
-                Opcode::IsInvalid => -1,
-                _ => unreachable!(),
-            };
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            ctx.emit(Inst::CmpRSImm16 {
-                op: CmpOp::CmpS64,
-                rn,
-                imm,
-            });
-
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let ty = ctx.output_ty(insn, 0);
-            lower_flags_to_bool_result(ctx, cond, rd, ty);
-        }
-
-        Opcode::Select => {
-            let ty = ctx.output_ty(insn, 0);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-            let cond = lower_boolean_to_flags(ctx, inputs[0]);
-            ctx.emit(Inst::gen_move(rd, rm, ty));
-            if ty_is_float(ty) {
-                if ty_bits(ty) < 64 {
-                    ctx.emit(Inst::FpuCMov32 { rd, cond, rm: rn });
-                } else {
-                    ctx.emit(Inst::FpuCMov64 { rd, cond, rm: rn });
-                }
-            } else {
-                if ty_bits(ty) < 64 {
-                    ctx.emit(Inst::CMov32 { rd, cond, rm: rn });
-                } else {
-                    ctx.emit(Inst::CMov64 { rd, cond, rm: rn });
-                }
-            }
-        }
-
-        Opcode::SelectifSpectreGuard => {
-            let ty = ctx.output_ty(insn, 0);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-            let condcode = ctx.data(insn).cond_code().unwrap();
-            let cond = Cond::from_intcc(condcode);
-            let is_signed = condcode_is_signed(condcode);
-
-            // Verification ensures that the input is always a single-def ifcmp.
-            let cmp_insn = ctx
-                .get_input_as_source_or_const(inputs[0].insn, inputs[0].input)
-                .inst
-                .unwrap()
-                .0;
-            debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
-            lower_icmp_to_flags(ctx, cmp_insn, is_signed, true);
-
-            ctx.emit(Inst::gen_move(rd, rm, ty));
-            if ty_is_float(ty) {
-                if ty_bits(ty) < 64 {
-                    ctx.emit(Inst::FpuCMov32 { rd, cond, rm: rn });
-                } else {
-                    ctx.emit(Inst::FpuCMov64 { rd, cond, rm: rn });
-                }
-            } else {
-                if ty_bits(ty) < 64 {
-                    ctx.emit(Inst::CMov32 { rd, cond, rm: rn });
-                } else {
-                    ctx.emit(Inst::CMov64 { rd, cond, rm: rn });
-                }
-            }
-        }
-
-        Opcode::Trap | Opcode::ResumableTrap => {
-            let trap_code = ctx.data(insn).trap_code().unwrap();
-            ctx.emit_safepoint(Inst::Trap { trap_code })
-        }
-
-        Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
-            let cond = lower_boolean_to_flags(ctx, inputs[0]);
-            let negated = op == Opcode::Trapz;
-            let cond = if negated { cond.invert() } else { cond };
-            let trap_code = ctx.data(insn).trap_code().unwrap();
-            ctx.emit_safepoint(Inst::TrapIf { trap_code, cond });
+        Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
+            let cond = lower_boolean_to_flags(ctx, inputs[0]);
+            let negated = op == Opcode::Trapz;
+            let cond = if negated { cond.invert() } else { cond };
+            let trap_code = ctx.data(insn).trap_code().unwrap();
+            ctx.emit_safepoint(Inst::TrapIf { trap_code, cond });
         }
 
         Opcode::Trapif => {
@@ -2681,161 +873,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             // N.B.: the Ret itself is generated by the ABI.
         }
 
-        Opcode::AtomicRmw => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let flags = ctx.memflags(insn).unwrap();
-            let endianness = flags.endianness(Endianness::Big);
-            let ty = ty.unwrap();
-            assert!(is_valid_atomic_transaction_ty(ty));
-            if endianness == Endianness::Little {
-                unimplemented!("Little-endian atomic operations not implemented");
-            }
-            if ty_bits(ty) < 32 {
-                unimplemented!("Sub-word atomic operations not implemented");
-            }
-            let op = inst_common::AtomicRmwOp::from(ctx.data(insn).atomic_rmw_op().unwrap());
-            let (alu_op, rn) = match op {
-                AtomicRmwOp::And => (choose_32_64(ty, ALUOp::And32, ALUOp::And64), rn),
-                AtomicRmwOp::Or => (choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64), rn),
-                AtomicRmwOp::Xor => (choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64), rn),
-                AtomicRmwOp::Add => (choose_32_64(ty, ALUOp::Add32, ALUOp::Add64), rn),
-                AtomicRmwOp::Sub => {
-                    let tmp_ty = choose_32_64(ty, types::I32, types::I64);
-                    let tmp = ctx.alloc_tmp(tmp_ty).only_reg().unwrap();
-                    let neg_op = choose_32_64(ty, UnaryOp::Neg32, UnaryOp::Neg64);
-                    ctx.emit(Inst::UnaryRR {
-                        op: neg_op,
-                        rd: tmp,
-                        rn,
-                    });
-                    (choose_32_64(ty, ALUOp::Add32, ALUOp::Add64), tmp.to_reg())
-                }
-                _ => unimplemented!("AtomicRmw operation type {:?} not implemented", op),
-            };
-            let mem = MemArg::reg(addr, flags);
-            ctx.emit(Inst::AtomicRmw {
-                alu_op,
-                rd,
-                rn,
-                mem,
-            });
-        }
-        Opcode::AtomicCas => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let addr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
-            let rn = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None);
-            let flags = ctx.memflags(insn).unwrap();
-            let endianness = flags.endianness(Endianness::Big);
-            let ty = ty.unwrap();
-            assert!(is_valid_atomic_transaction_ty(ty));
-            if endianness == Endianness::Little {
-                unimplemented!("Little-endian atomic operations not implemented");
-            }
-            if ty_bits(ty) < 32 {
-                unimplemented!("Sub-word atomic operations not implemented");
-            }
-            let mem = MemArg::reg(addr, flags);
-            ctx.emit(Inst::gen_move(rd, rm, ty));
-            if ty_bits(ty) == 32 {
-                ctx.emit(Inst::AtomicCas32 { rd, rn, mem });
-            } else {
-                ctx.emit(Inst::AtomicCas64 { rd, rn, mem });
-            }
-        }
-        Opcode::AtomicLoad => {
-            let flags = ctx.memflags(insn).unwrap();
-            let endianness = flags.endianness(Endianness::Big);
-            let ty = ty.unwrap();
-            assert!(is_valid_atomic_transaction_ty(ty));
-
-            let mem = lower_address(ctx, &inputs[..], 0, flags);
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-
-            if endianness == Endianness::Big {
-                ctx.emit(match ty_bits(ty) {
-                    8 => Inst::Load32ZExt8 { rd, mem },
-                    16 => Inst::Load32ZExt16 { rd, mem },
-                    32 => Inst::Load32 { rd, mem },
-                    64 => Inst::Load64 { rd, mem },
-                    _ => panic!("Unsupported size in load"),
-                });
-            } else {
-                ctx.emit(match ty_bits(ty) {
-                    8 => Inst::Load32ZExt8 { rd, mem },
-                    16 => Inst::LoadRev16 { rd, mem },
-                    32 => Inst::LoadRev32 { rd, mem },
-                    64 => Inst::LoadRev64 { rd, mem },
-                    _ => panic!("Unsupported size in load"),
-                });
-            }
-        }
-        Opcode::AtomicStore => {
-            let flags = ctx.memflags(insn).unwrap();
-            let endianness = flags.endianness(Endianness::Big);
-            let ty = ctx.input_ty(insn, 0);
-            assert!(is_valid_atomic_transaction_ty(ty));
-
-            let mem = lower_address(ctx, &inputs[1..], 0, flags);
-
-            if ty_bits(ty) <= 16 {
-                if let Some(imm) = input_matches_const(ctx, inputs[0]) {
-                    ctx.emit(match (endianness, ty_bits(ty)) {
-                        (_, 8) => Inst::StoreImm8 {
-                            imm: imm as u8,
-                            mem,
-                        },
-                        (Endianness::Big, 16) => Inst::StoreImm16 {
-                            imm: imm as i16,
-                            mem,
-                        },
-                        (Endianness::Little, 16) => Inst::StoreImm16 {
-                            imm: (imm as i16).swap_bytes(),
-                            mem,
-                        },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                } else {
-                    let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                    ctx.emit(match (endianness, ty_bits(ty)) {
-                        (_, 8) => Inst::Store8 { rd, mem },
-                        (Endianness::Big, 16) => Inst::Store16 { rd, mem },
-                        (Endianness::Little, 16) => Inst::StoreRev16 { rd, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                }
-            } else if endianness == Endianness::Big {
-                if let Some(imm) = input_matches_simm16(ctx, inputs[0]) {
-                    ctx.emit(match ty_bits(ty) {
-                        32 => Inst::StoreImm32SExt16 { imm, mem },
-                        64 => Inst::StoreImm64SExt16 { imm, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                } else {
-                    let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                    ctx.emit(match ty_bits(ty) {
-                        32 => Inst::Store32 { rd, mem },
-                        64 => Inst::Store64 { rd, mem },
-                        _ => panic!("Unsupported size in store"),
-                    });
-                }
-            } else {
-                let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                ctx.emit(match ty_bits(ty) {
-                    32 => Inst::StoreRev32 { rd, mem },
-                    64 => Inst::StoreRev64 { rd, mem },
-                    _ => panic!("Unsupported size in store"),
-                });
-            }
-
-            ctx.emit(Inst::Fence);
-        }
-        Opcode::Fence => {
-            ctx.emit(Inst::Fence);
-        }
-
         Opcode::RawBitcast
         | Opcode::Splat
         | Opcode::Swizzle
@@ -2891,6 +928,20 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             panic!("Unused opcode should not be encountered.");
         }
 
+        Opcode::LoadComplex
+        | Opcode::Uload8Complex
+        | Opcode::Sload8Complex
+        | Opcode::Uload16Complex
+        | Opcode::Sload16Complex
+        | Opcode::Uload32Complex
+        | Opcode::Sload32Complex
+        | Opcode::StoreComplex
+        | Opcode::Istore8Complex
+        | Opcode::Istore16Complex
+        | Opcode::Istore32Complex => {
+            panic!("Load/store complex opcode should not be encountered.");
+        }
+
         Opcode::Ifcmp
         | Opcode::Ffcmp
         | Opcode::Trapff
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs
new file mode 100644
index 000000000000..d4dcc0b5556e
--- /dev/null
+++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs
@@ -0,0 +1,475 @@
+//! ISLE integration glue code for s390x lowering.
+
+// Pull in the ISLE generated code.
+pub mod generated_code;
+
+// Types that the generated ISLE code uses via `use super::*`.
+use super::{
+    BranchTarget, CallIndInfo, CallInfo, Cond, Inst as MInst, JTSequenceInfo, MachLabel, MemArg,
+    MemFlags, Opcode, Reg, UImm16Shifted, UImm32Shifted,
+};
+use crate::isa::s390x::settings::Flags as IsaFlags;
+use crate::machinst::isle::*;
+use crate::settings::Flags;
+use crate::{
+    ir::{
+        condcodes::*, immediates::*, types::*, AtomicRmwOp, Endianness, ExternalName, Inst,
+        InstructionData, StackSlot, TrapCode, Value, ValueLabel, ValueList,
+    },
+    isa::s390x::inst::s390x_map_regs,
+    isa::unwind::UnwindInst,
+    machinst::{InsnOutput, LowerCtx, RelocDistance},
+};
+use std::boxed::Box;
+use std::convert::TryFrom;
+use std::vec::Vec;
+
+type BoxCallInfo = Box<CallInfo>;
+type BoxCallIndInfo = Box<CallIndInfo>;
+type VecMachLabel = Vec<MachLabel>;
+type BoxJTSequenceInfo = Box<JTSequenceInfo>;
+type BoxExternalName = Box<ExternalName>;
+
+/// The main entry point for lowering with ISLE.
+pub(crate) fn lower<C>(
+    lower_ctx: &mut C,
+    flags: &Flags,
+    isa_flags: &IsaFlags,
+    outputs: &[InsnOutput],
+    inst: Inst,
+) -> Result<(), ()>
+where
+    C: LowerCtx<I = MInst>,
+{
+    lower_common(
+        lower_ctx,
+        flags,
+        isa_flags,
+        outputs,
+        inst,
+        |cx, insn| generated_code::constructor_lower(cx, insn),
+        s390x_map_regs,
+    )
+}
+
+impl<C> generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6>
+where
+    C: LowerCtx<I = MInst>,
+{
+    isle_prelude_methods!();
+
+    #[inline]
+    fn allow_div_traps(&mut self, _: Type) -> Option<()> {
+        if !self.flags.avoid_div_traps() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn mie2_enabled(&mut self, _: Type) -> Option<()> {
+        if self.isa_flags.has_mie2() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn mie2_disabled(&mut self, _: Type) -> Option<()> {
+        if !self.isa_flags.has_mie2() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn vxrs_ext2_enabled(&mut self, _: Type) -> Option<()> {
+        if self.isa_flags.has_vxrs_ext2() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn vxrs_ext2_disabled(&mut self, _: Type) -> Option<()> {
+        if !self.isa_flags.has_vxrs_ext2() {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn symbol_value_data(&mut self, inst: Inst) -> Option<(BoxExternalName, RelocDistance, i64)> {
+        let (name, dist, offset) = self.lower_ctx.symbol_value(inst)?;
+        Some((Box::new(name.clone()), dist, offset))
+    }
+
+    #[inline]
+    fn call_target_data(&mut self, inst: Inst) -> Option<(BoxExternalName, RelocDistance)> {
+        let (name, dist) = self.lower_ctx.call_target(inst)?;
+        Some((Box::new(name.clone()), dist))
+    }
+
+    #[inline]
+    fn writable_gpr(&mut self, regno: u8) -> WritableReg {
+        super::writable_gpr(regno)
+    }
+
+    #[inline]
+    fn zero_reg(&mut self) -> Reg {
+        super::zero_reg()
+    }
+
+    #[inline]
+    fn gpr32_ty(&mut self, ty: Type) -> Option<Type> {
+        match ty {
+            I8 | I16 | I32 | B1 | B8 | B16 | B32 => Some(ty),
+            _ => None,
+        }
+    }
+
+    #[inline]
+    fn gpr64_ty(&mut self, ty: Type) -> Option<Type> {
+        match ty {
+            I64 | B64 | R64 => Some(ty),
+            _ => None,
+        }
+    }
+
+    #[inline]
+    fn uimm32shifted(&mut self, n: u32, shift: u8) -> UImm32Shifted {
+        UImm32Shifted::maybe_with_shift(n, shift).unwrap()
+    }
+
+    #[inline]
+    fn uimm16shifted(&mut self, n: u16, shift: u8) -> UImm16Shifted {
+        UImm16Shifted::maybe_with_shift(n, shift).unwrap()
+    }
+
+    #[inline]
+    fn i64_nonequal(&mut self, val: i64, cmp: i64) -> Option<i64> {
+        if val != cmp {
+            Some(val)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn u8_as_u16(&mut self, n: u8) -> u16 {
+        n as u16
+    }
+
+    #[inline]
+    fn u64_as_u32(&mut self, n: u64) -> u32 {
+        n as u32
+    }
+
+    #[inline]
+    fn u64_as_i16(&mut self, n: u64) -> i16 {
+        n as i16
+    }
+
+    #[inline]
+    fn u64_nonzero_hipart(&mut self, n: u64) -> Option<u64> {
+        let part = n & 0xffff_ffff_0000_0000;
+        if part != 0 {
+            Some(part)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn u64_nonzero_lopart(&mut self, n: u64) -> Option<u64> {
+        let part = n & 0x0000_0000_ffff_ffff;
+        if part != 0 {
+            Some(part)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn i32_from_u64(&mut self, n: u64) -> Option<i32> {
+        if let Ok(imm) = i32::try_from(n as i64) {
+            Some(imm)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn i16_from_u64(&mut self, n: u64) -> Option<i16> {
+        if let Ok(imm) = i16::try_from(n as i64) {
+            Some(imm)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn uimm32shifted_from_u64(&mut self, n: u64) -> Option<UImm32Shifted> {
+        UImm32Shifted::maybe_from_u64(n)
+    }
+
+    #[inline]
+    fn uimm16shifted_from_u64(&mut self, n: u64) -> Option<UImm16Shifted> {
+        UImm16Shifted::maybe_from_u64(n)
+    }
+
+    #[inline]
+    fn u64_from_value(&mut self, val: Value) -> Option<u64> {
+        let inst = self.lower_ctx.dfg().value_def(val).inst()?;
+        let constant = self.lower_ctx.get_constant(inst)?;
+        Some(constant)
+    }
+
+    #[inline]
+    fn u32_from_value(&mut self, val: Value) -> Option<u32> {
+        let constant = self.u64_from_value(val)?;
+        let imm = u32::try_from(constant).ok()?;
+        Some(imm)
+    }
+
+    #[inline]
+    fn u8_from_value(&mut self, val: Value) -> Option<u8> {
+        let constant = self.u64_from_value(val)?;
+        let imm = u8::try_from(constant).ok()?;
+        Some(imm)
+    }
+
+    #[inline]
+    fn u64_from_signed_value(&mut self, val: Value) -> Option<u64> {
+        let inst = self.lower_ctx.dfg().value_def(val).inst()?;
+        let constant = self.lower_ctx.get_constant(inst)?;
+        let ty = self.lower_ctx.output_ty(inst, 0);
+        Some(super::sign_extend_to_u64(constant, self.ty_bits(ty)))
+    }
+
+    #[inline]
+    fn i64_from_value(&mut self, val: Value) -> Option<i64> {
+        let constant = self.u64_from_signed_value(val)? as i64;
+        Some(constant)
+    }
+
+    #[inline]
+    fn i32_from_value(&mut self, val: Value) -> Option<i32> {
+        let constant = self.u64_from_signed_value(val)? as i64;
+        let imm = i32::try_from(constant).ok()?;
+        Some(imm)
+    }
+
+    #[inline]
+    fn i16_from_value(&mut self, val: Value) -> Option<i16> {
+        let constant = self.u64_from_signed_value(val)? as i64;
+        let imm = i16::try_from(constant).ok()?;
+        Some(imm)
+    }
+
+    #[inline]
+    fn i16_from_swapped_value(&mut self, val: Value) -> Option<i16> {
+        let constant = self.u64_from_signed_value(val)? as i64;
+        let imm = i16::try_from(constant).ok()?;
+        Some(imm.swap_bytes())
+    }
+
+    #[inline]
+    fn i64_from_negated_value(&mut self, val: Value) -> Option<i64> {
+        let constant = self.u64_from_signed_value(val)? as i64;
+        let imm = -constant;
+        Some(imm)
+    }
+
+    #[inline]
+    fn i32_from_negated_value(&mut self, val: Value) -> Option<i32> {
+        let constant = self.u64_from_signed_value(val)? as i64;
+        let imm = i32::try_from(-constant).ok()?;
+        Some(imm)
+    }
+
+    #[inline]
+    fn i16_from_negated_value(&mut self, val: Value) -> Option<i16> {
+        let constant = self.u64_from_signed_value(val)? as i64;
+        let imm = i16::try_from(-constant).ok()?;
+        Some(imm)
+    }
+
+    #[inline]
+    fn uimm16shifted_from_value(&mut self, val: Value) -> Option<UImm16Shifted> {
+        let constant = self.u64_from_value(val)?;
+        UImm16Shifted::maybe_from_u64(constant)
+    }
+
+    #[inline]
+    fn uimm32shifted_from_value(&mut self, val: Value) -> Option<UImm32Shifted> {
+        let constant = self.u64_from_value(val)?;
+        UImm32Shifted::maybe_from_u64(constant)
+    }
+
+    #[inline]
+    fn uimm16shifted_from_inverted_value(&mut self, val: Value) -> Option<UImm16Shifted> {
+        let constant = self.u64_from_value(val)?;
+        let imm = UImm16Shifted::maybe_from_u64(!constant)?;
+        Some(imm.negate_bits())
+    }
+
+    #[inline]
+    fn uimm32shifted_from_inverted_value(&mut self, val: Value) -> Option<UImm32Shifted> {
+        let constant = self.u64_from_value(val)?;
+        let imm = UImm32Shifted::maybe_from_u64(!constant)?;
+        Some(imm.negate_bits())
+    }
+
+    #[inline]
+    fn mask_amt_imm(&mut self, ty: Type, amt: i64) -> u8 {
+        let mask = self.ty_bits(ty) - 1;
+        (amt as u8) & mask
+    }
+
+    #[inline]
+    fn mask_as_cond(&mut self, mask: u8) -> Cond {
+        Cond::from_mask(mask)
+    }
+
+    #[inline]
+    fn intcc_as_cond(&mut self, cc: &IntCC) -> Cond {
+        Cond::from_intcc(*cc)
+    }
+
+    #[inline]
+    fn floatcc_as_cond(&mut self, cc: &FloatCC) -> Cond {
+        Cond::from_floatcc(*cc)
+    }
+
+    #[inline]
+    fn invert_cond(&mut self, cond: &Cond) -> Cond {
+        Cond::invert(*cond)
+    }
+
+    #[inline]
+    fn signed(&mut self, cc: &IntCC) -> Option<()> {
+        if super::condcode_is_signed(*cc) {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn unsigned(&mut self, cc: &IntCC) -> Option<()> {
+        if !super::condcode_is_signed(*cc) {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn reloc_distance_near(&mut self, dist: &RelocDistance) -> Option<()> {
+        if *dist == RelocDistance::Near {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn zero_offset(&mut self) -> Offset32 {
+        Offset32::new(0)
+    }
+
+    #[inline]
+    fn i64_from_offset(&mut self, off: Offset32) -> i64 {
+        i64::from(off)
+    }
+
+    #[inline]
+    fn littleendian(&mut self, flags: MemFlags) -> Option<()> {
+        let endianness = flags.endianness(Endianness::Big);
+        if endianness == Endianness::Little {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn bigendian(&mut self, flags: MemFlags) -> Option<()> {
+        let endianness = flags.endianness(Endianness::Big);
+        if endianness == Endianness::Big {
+            Some(())
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn memflags_trusted(&mut self) -> MemFlags {
+        MemFlags::trusted()
+    }
+
+    #[inline]
+    fn memarg_reg_plus_reg(&mut self, x: Reg, y: Reg, flags: MemFlags) -> MemArg {
+        MemArg::reg_plus_reg(x, y, flags)
+    }
+
+    #[inline]
+    fn memarg_reg_plus_off(&mut self, reg: Reg, off: i64, flags: MemFlags) -> MemArg {
+        MemArg::reg_plus_off(reg, off, flags)
+    }
+
+    #[inline]
+    fn memarg_symbol(&mut self, name: BoxExternalName, offset: i32, flags: MemFlags) -> MemArg {
+        MemArg::Symbol {
+            name,
+            offset,
+            flags,
+        }
+    }
+
+    #[inline]
+    fn memarg_symbol_offset_sum(&mut self, off1: i64, off2: i64) -> Option<i32> {
+        let off = i32::try_from(off1 + off2).ok()?;
+        if off & 1 == 0 {
+            Some(off)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn abi_stackslot_addr(
+        &mut self,
+        dst: WritableReg,
+        stack_slot: StackSlot,
+        offset: Offset32,
+    ) -> MInst {
+        let offset = u32::try_from(i32::from(offset)).unwrap();
+        self.lower_ctx.abi().stackslot_addr(stack_slot, offset, dst)
+    }
+
+    #[inline]
+    fn sinkable_inst(&mut self, val: Value) -> Option<Inst> {
+        let input = self.lower_ctx.get_value_as_source_or_const(val);
+        if let Some((inst, 0)) = input.inst {
+            return Some(inst);
+        }
+        None
+    }
+
+    #[inline]
+    fn sink_inst(&mut self, inst: Inst) -> Unit {
+        self.lower_ctx.sink_inst(inst);
+    }
+
+    #[inline]
+    fn emit(&mut self, inst: &MInst) -> Unit {
+        self.emitted_insts.push(inst.clone());
+    }
+}
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
new file mode 100644
index 000000000000..202397176c41
--- /dev/null
+++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
@@ -0,0 +1,4 @@
+src/clif.isle f176ef3bba99365
+src/prelude.isle 51d2aef2566c1c96
+src/isa/s390x/inst.isle 63cf833b5cfd727d
+src/isa/s390x/lower.isle a0e21a567040bc33
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs
new file mode 100644
index 000000000000..1103e01d860b
--- /dev/null
+++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs
@@ -0,0 +1,11438 @@
+// GENERATED BY ISLE. DO NOT EDIT!
+//
+// Generated automatically from the instruction-selection DSL code in:
+// - src/clif.isle
+// - src/prelude.isle
+// - src/isa/s390x/inst.isle
+// - src/isa/s390x/lower.isle
+
+#![allow(dead_code, unreachable_code, unreachable_patterns)]
+#![allow(unused_imports, unused_variables, non_snake_case)]
+#![allow(irrefutable_let_patterns)]
+
+use super::*; // Pulls in all external types.
+
+/// Context during lowering: an implementation of this trait
+/// must be provided with all external constructors and extractors.
+/// A mutable borrow is passed along through all lowering logic.
+pub trait Context {
+    fn unpack_value_array_2(&mut self, arg0: &ValueArray2) -> (Value, Value);
+    fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> ValueArray2;
+    fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value);
+    fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3;
+    fn u32_add(&mut self, arg0: u32, arg1: u32) -> u32;
+    fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8;
+    fn value_reg(&mut self, arg0: Reg) -> ValueRegs;
+    fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs;
+    fn value_regs_invalid(&mut self) -> ValueRegs;
+    fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg;
+    fn invalid_reg(&mut self) -> Reg;
+    fn put_in_reg(&mut self, arg0: Value) -> Reg;
+    fn put_in_regs(&mut self, arg0: Value) -> ValueRegs;
+    fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg;
+    fn u8_as_u64(&mut self, arg0: u8) -> u64;
+    fn u16_as_u64(&mut self, arg0: u16) -> u64;
+    fn u32_as_u64(&mut self, arg0: u32) -> u64;
+    fn ty_bits(&mut self, arg0: Type) -> u8;
+    fn ty_bits_u16(&mut self, arg0: Type) -> u16;
+    fn ty_bytes(&mut self, arg0: Type) -> u16;
+    fn lane_type(&mut self, arg0: Type) -> Type;
+    fn fits_in_16(&mut self, arg0: Type) -> Option<Type>;
+    fn fits_in_32(&mut self, arg0: Type) -> Option<Type>;
+    fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
+    fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
+    fn vec128(&mut self, arg0: Type) -> Option<Type>;
+    fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
+    fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
+    fn unwrap_head_value_list_1(&mut self, arg0: ValueList) -> (Value, ValueSlice);
+    fn unwrap_head_value_list_2(&mut self, arg0: ValueList) -> (Value, Value, ValueSlice);
+    fn writable_reg_to_reg(&mut self, arg0: WritableReg) -> Reg;
+    fn u8_from_uimm8(&mut self, arg0: Uimm8) -> u8;
+    fn u64_from_imm64(&mut self, arg0: Imm64) -> u64;
+    fn nonzero_u64_from_imm64(&mut self, arg0: Imm64) -> Option<u64>;
+    fn u64_from_ieee32(&mut self, arg0: Ieee32) -> u64;
+    fn u64_from_ieee64(&mut self, arg0: Ieee64) -> u64;
+    fn inst_results(&mut self, arg0: Inst) -> ValueSlice;
+    fn first_result(&mut self, arg0: Inst) -> Option<Value>;
+    fn inst_data(&mut self, arg0: Inst) -> InstructionData;
+    fn value_type(&mut self, arg0: Value) -> Type;
+    fn multi_lane(&mut self, arg0: Type) -> Option<(u8, u16)>;
+    fn def_inst(&mut self, arg0: Value) -> Option<Inst>;
+    fn trap_code_division_by_zero(&mut self) -> TrapCode;
+    fn trap_code_integer_overflow(&mut self) -> TrapCode;
+    fn trap_code_bad_conversion_to_integer(&mut self) -> TrapCode;
+    fn avoid_div_traps(&mut self, arg0: Type) -> Option<()>;
+    fn mie2_enabled(&mut self, arg0: Type) -> Option<()>;
+    fn mie2_disabled(&mut self, arg0: Type) -> Option<()>;
+    fn vxrs_ext2_enabled(&mut self, arg0: Type) -> Option<()>;
+    fn vxrs_ext2_disabled(&mut self, arg0: Type) -> Option<()>;
+    fn allow_div_traps(&mut self, arg0: Type) -> Option<()>;
+    fn symbol_value_data(&mut self, arg0: Inst) -> Option<(BoxExternalName, RelocDistance, i64)>;
+    fn call_target_data(&mut self, arg0: Inst) -> Option<(BoxExternalName, RelocDistance)>;
+    fn writable_gpr(&mut self, arg0: u8) -> WritableReg;
+    fn zero_reg(&mut self) -> Reg;
+    fn gpr32_ty(&mut self, arg0: Type) -> Option<Type>;
+    fn gpr64_ty(&mut self, arg0: Type) -> Option<Type>;
+    fn uimm32shifted(&mut self, arg0: u32, arg1: u8) -> UImm32Shifted;
+    fn uimm16shifted(&mut self, arg0: u16, arg1: u8) -> UImm16Shifted;
+    fn i64_nonequal(&mut self, arg0: i64, arg1: i64) -> Option<i64>;
+    fn u8_as_u16(&mut self, arg0: u8) -> u16;
+    fn u64_as_u32(&mut self, arg0: u64) -> u32;
+    fn u64_as_i16(&mut self, arg0: u64) -> i16;
+    fn u64_nonzero_hipart(&mut self, arg0: u64) -> Option<u64>;
+    fn u64_nonzero_lopart(&mut self, arg0: u64) -> Option<u64>;
+    fn i32_from_u64(&mut self, arg0: u64) -> Option<i32>;
+    fn i16_from_u64(&mut self, arg0: u64) -> Option<i16>;
+    fn uimm32shifted_from_u64(&mut self, arg0: u64) -> Option<UImm32Shifted>;
+    fn uimm16shifted_from_u64(&mut self, arg0: u64) -> Option<UImm16Shifted>;
+    fn u64_from_value(&mut self, arg0: Value) -> Option<u64>;
+    fn u32_from_value(&mut self, arg0: Value) -> Option<u32>;
+    fn u8_from_value(&mut self, arg0: Value) -> Option<u8>;
+    fn u64_from_signed_value(&mut self, arg0: Value) -> Option<u64>;
+    fn i64_from_value(&mut self, arg0: Value) -> Option<i64>;
+    fn i32_from_value(&mut self, arg0: Value) -> Option<i32>;
+    fn i16_from_value(&mut self, arg0: Value) -> Option<i16>;
+    fn i16_from_swapped_value(&mut self, arg0: Value) -> Option<i16>;
+    fn i64_from_negated_value(&mut self, arg0: Value) -> Option<i64>;
+    fn i32_from_negated_value(&mut self, arg0: Value) -> Option<i32>;
+    fn i16_from_negated_value(&mut self, arg0: Value) -> Option<i16>;
+    fn uimm16shifted_from_value(&mut self, arg0: Value) -> Option<UImm16Shifted>;
+    fn uimm32shifted_from_value(&mut self, arg0: Value) -> Option<UImm32Shifted>;
+    fn uimm16shifted_from_inverted_value(&mut self, arg0: Value) -> Option<UImm16Shifted>;
+    fn uimm32shifted_from_inverted_value(&mut self, arg0: Value) -> Option<UImm32Shifted>;
+    fn mask_amt_imm(&mut self, arg0: Type, arg1: i64) -> u8;
+    fn mask_as_cond(&mut self, arg0: u8) -> Cond;
+    fn intcc_as_cond(&mut self, arg0: &IntCC) -> Cond;
+    fn floatcc_as_cond(&mut self, arg0: &FloatCC) -> Cond;
+    fn invert_cond(&mut self, arg0: &Cond) -> Cond;
+    fn signed(&mut self, arg0: &IntCC) -> Option<()>;
+    fn unsigned(&mut self, arg0: &IntCC) -> Option<()>;
+    fn reloc_distance_near(&mut self, arg0: &RelocDistance) -> Option<()>;
+    fn zero_offset(&mut self) -> Offset32;
+    fn i64_from_offset(&mut self, arg0: Offset32) -> i64;
+    fn littleendian(&mut self, arg0: MemFlags) -> Option<()>;
+    fn bigendian(&mut self, arg0: MemFlags) -> Option<()>;
+    fn memflags_trusted(&mut self) -> MemFlags;
+    fn memarg_reg_plus_reg(&mut self, arg0: Reg, arg1: Reg, arg2: MemFlags) -> MemArg;
+    fn memarg_reg_plus_off(&mut self, arg0: Reg, arg1: i64, arg2: MemFlags) -> MemArg;
+    fn memarg_symbol(&mut self, arg0: BoxExternalName, arg1: i32, arg2: MemFlags) -> MemArg;
+    fn memarg_symbol_offset_sum(&mut self, arg0: i64, arg1: i64) -> Option<i32>;
+    fn abi_stackslot_addr(&mut self, arg0: WritableReg, arg1: StackSlot, arg2: Offset32) -> MInst;
+    fn sinkable_inst(&mut self, arg0: Value) -> Option<Inst>;
+    fn sink_inst(&mut self, arg0: Inst) -> Unit;
+    fn emit(&mut self, arg0: &MInst) -> Unit;
+}
+
+/// Internal type SideEffectNoResult: defined at src/prelude.isle line 282.
+#[derive(Clone, Debug)]
+pub enum SideEffectNoResult {
+    Inst { inst: MInst },
+}
+
+/// Internal type ProducesFlags: defined at src/prelude.isle line 295.
+#[derive(Clone, Debug)]
+pub enum ProducesFlags {
+    ProducesFlags { inst: MInst, result: Reg },
+}
+
+/// Internal type ConsumesFlags: defined at src/prelude.isle line 298.
+#[derive(Clone, Debug)]
+pub enum ConsumesFlags {
+    ConsumesFlags { inst: MInst, result: Reg },
+}
+
+/// Internal type MInst: defined at src/isa/s390x/inst.isle line 2.
+#[derive(Clone, Debug)]
+pub enum MInst {
+    Nop0,
+    Nop2,
+    AluRRR {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        rn: Reg,
+        rm: Reg,
+    },
+    AluRRSImm16 {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        rn: Reg,
+        imm: i16,
+    },
+    AluRR {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        rm: Reg,
+    },
+    AluRX {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    AluRSImm16 {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        imm: i16,
+    },
+    AluRSImm32 {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        imm: i32,
+    },
+    AluRUImm32 {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        imm: u32,
+    },
+    AluRUImm16Shifted {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        imm: UImm16Shifted,
+    },
+    AluRUImm32Shifted {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        imm: UImm32Shifted,
+    },
+    SMulWide {
+        rn: Reg,
+        rm: Reg,
+    },
+    UMulWide {
+        rn: Reg,
+    },
+    SDivMod32 {
+        rn: Reg,
+    },
+    SDivMod64 {
+        rn: Reg,
+    },
+    UDivMod32 {
+        rn: Reg,
+    },
+    UDivMod64 {
+        rn: Reg,
+    },
+    Flogr {
+        rn: Reg,
+    },
+    ShiftRR {
+        shift_op: ShiftOp,
+        rd: WritableReg,
+        rn: Reg,
+        shift_imm: u8,
+        shift_reg: Reg,
+    },
+    UnaryRR {
+        op: UnaryOp,
+        rd: WritableReg,
+        rn: Reg,
+    },
+    CmpRR {
+        op: CmpOp,
+        rn: Reg,
+        rm: Reg,
+    },
+    CmpRX {
+        op: CmpOp,
+        rn: Reg,
+        mem: MemArg,
+    },
+    CmpRSImm16 {
+        op: CmpOp,
+        rn: Reg,
+        imm: i16,
+    },
+    CmpRSImm32 {
+        op: CmpOp,
+        rn: Reg,
+        imm: i32,
+    },
+    CmpRUImm32 {
+        op: CmpOp,
+        rn: Reg,
+        imm: u32,
+    },
+    CmpTrapRR {
+        op: CmpOp,
+        rn: Reg,
+        rm: Reg,
+        cond: Cond,
+        trap_code: TrapCode,
+    },
+    CmpTrapRSImm16 {
+        op: CmpOp,
+        rn: Reg,
+        imm: i16,
+        cond: Cond,
+        trap_code: TrapCode,
+    },
+    CmpTrapRUImm16 {
+        op: CmpOp,
+        rn: Reg,
+        imm: u16,
+        cond: Cond,
+        trap_code: TrapCode,
+    },
+    AtomicRmw {
+        alu_op: ALUOp,
+        rd: WritableReg,
+        rn: Reg,
+        mem: MemArg,
+    },
+    AtomicCas32 {
+        rd: WritableReg,
+        rn: Reg,
+        mem: MemArg,
+    },
+    AtomicCas64 {
+        rd: WritableReg,
+        rn: Reg,
+        mem: MemArg,
+    },
+    Fence,
+    Load32 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load32ZExt8 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load32SExt8 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load32ZExt16 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load32SExt16 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load64 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load64ZExt8 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load64SExt8 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load64ZExt16 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load64SExt16 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load64ZExt32 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Load64SExt32 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    LoadRev16 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    LoadRev32 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    LoadRev64 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    Store8 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    Store16 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    Store32 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    Store64 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    StoreImm8 {
+        imm: u8,
+        mem: MemArg,
+    },
+    StoreImm16 {
+        imm: i16,
+        mem: MemArg,
+    },
+    StoreImm32SExt16 {
+        imm: i16,
+        mem: MemArg,
+    },
+    StoreImm64SExt16 {
+        imm: i16,
+        mem: MemArg,
+    },
+    StoreRev16 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    StoreRev32 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    StoreRev64 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    LoadMultiple64 {
+        rt: WritableReg,
+        rt2: WritableReg,
+        mem: MemArg,
+    },
+    StoreMultiple64 {
+        rt: Reg,
+        rt2: Reg,
+        mem: MemArg,
+    },
+    Mov32 {
+        rd: WritableReg,
+        rm: Reg,
+    },
+    Mov64 {
+        rd: WritableReg,
+        rm: Reg,
+    },
+    Mov32Imm {
+        rd: WritableReg,
+        imm: u32,
+    },
+    Mov32SImm16 {
+        rd: WritableReg,
+        imm: i16,
+    },
+    Mov64SImm16 {
+        rd: WritableReg,
+        imm: i16,
+    },
+    Mov64SImm32 {
+        rd: WritableReg,
+        imm: i32,
+    },
+    Mov64UImm16Shifted {
+        rd: WritableReg,
+        imm: UImm16Shifted,
+    },
+    Mov64UImm32Shifted {
+        rd: WritableReg,
+        imm: UImm32Shifted,
+    },
+    Insert64UImm16Shifted {
+        rd: WritableReg,
+        imm: UImm16Shifted,
+    },
+    Insert64UImm32Shifted {
+        rd: WritableReg,
+        imm: UImm32Shifted,
+    },
+    Extend {
+        rd: WritableReg,
+        rn: Reg,
+        signed: bool,
+        from_bits: u8,
+        to_bits: u8,
+    },
+    CMov32 {
+        rd: WritableReg,
+        cond: Cond,
+        rm: Reg,
+    },
+    CMov64 {
+        rd: WritableReg,
+        cond: Cond,
+        rm: Reg,
+    },
+    CMov32SImm16 {
+        rd: WritableReg,
+        cond: Cond,
+        imm: i16,
+    },
+    CMov64SImm16 {
+        rd: WritableReg,
+        cond: Cond,
+        imm: i16,
+    },
+    FpuMove32 {
+        rd: WritableReg,
+        rn: Reg,
+    },
+    FpuMove64 {
+        rd: WritableReg,
+        rn: Reg,
+    },
+    FpuCMov32 {
+        rd: WritableReg,
+        cond: Cond,
+        rm: Reg,
+    },
+    FpuCMov64 {
+        rd: WritableReg,
+        cond: Cond,
+        rm: Reg,
+    },
+    MovToFpr {
+        rd: WritableReg,
+        rn: Reg,
+    },
+    MovFromFpr {
+        rd: WritableReg,
+        rn: Reg,
+    },
+    FpuRR {
+        fpu_op: FPUOp1,
+        rd: WritableReg,
+        rn: Reg,
+    },
+    FpuRRR {
+        fpu_op: FPUOp2,
+        rd: WritableReg,
+        rm: Reg,
+    },
+    FpuRRRR {
+        fpu_op: FPUOp3,
+        rd: WritableReg,
+        rn: Reg,
+        rm: Reg,
+    },
+    FpuCopysign {
+        rd: WritableReg,
+        rn: Reg,
+        rm: Reg,
+    },
+    FpuCmp32 {
+        rn: Reg,
+        rm: Reg,
+    },
+    FpuCmp64 {
+        rn: Reg,
+        rm: Reg,
+    },
+    FpuLoad32 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    FpuStore32 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    FpuLoad64 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    FpuStore64 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    FpuLoadRev32 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    FpuStoreRev32 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    FpuLoadRev64 {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    FpuStoreRev64 {
+        rd: Reg,
+        mem: MemArg,
+    },
+    LoadFpuConst32 {
+        rd: WritableReg,
+        const_data: u32,
+    },
+    LoadFpuConst64 {
+        rd: WritableReg,
+        const_data: u64,
+    },
+    FpuToInt {
+        op: FpuToIntOp,
+        rd: WritableReg,
+        rn: Reg,
+    },
+    IntToFpu {
+        op: IntToFpuOp,
+        rd: WritableReg,
+        rn: Reg,
+    },
+    FpuRound {
+        op: FpuRoundMode,
+        rd: WritableReg,
+        rn: Reg,
+    },
+    FpuVecRRR {
+        fpu_op: FPUOp2,
+        rd: WritableReg,
+        rn: Reg,
+        rm: Reg,
+    },
+    Call {
+        link: WritableReg,
+        info: BoxCallInfo,
+    },
+    CallInd {
+        link: WritableReg,
+        info: BoxCallIndInfo,
+    },
+    Ret {
+        link: Reg,
+    },
+    EpiloguePlaceholder,
+    Jump {
+        dest: BranchTarget,
+    },
+    CondBr {
+        taken: BranchTarget,
+        not_taken: BranchTarget,
+        cond: Cond,
+    },
+    TrapIf {
+        cond: Cond,
+        trap_code: TrapCode,
+    },
+    OneWayCondBr {
+        target: BranchTarget,
+        cond: Cond,
+    },
+    IndirectBr {
+        rn: Reg,
+        targets: VecMachLabel,
+    },
+    Debugtrap,
+    Trap {
+        trap_code: TrapCode,
+    },
+    JTSequence {
+        info: BoxJTSequenceInfo,
+        ridx: Reg,
+        rtmp1: WritableReg,
+        rtmp2: WritableReg,
+    },
+    LoadExtNameFar {
+        rd: WritableReg,
+        name: BoxExternalName,
+        offset: i64,
+    },
+    LoadAddr {
+        rd: WritableReg,
+        mem: MemArg,
+    },
+    VirtualSPOffsetAdj {
+        offset: i64,
+    },
+    ValueLabelMarker {
+        reg: Reg,
+        label: ValueLabel,
+    },
+    Unwind {
+        inst: UnwindInst,
+    },
+}
+
+/// Internal type ALUOp: defined at src/isa/s390x/inst.isle line 691.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum ALUOp {
+    Add32,
+    Add32Ext16,
+    Add64,
+    Add64Ext16,
+    Add64Ext32,
+    AddLogical32,
+    AddLogical64,
+    AddLogical64Ext32,
+    Sub32,
+    Sub32Ext16,
+    Sub64,
+    Sub64Ext16,
+    Sub64Ext32,
+    SubLogical32,
+    SubLogical64,
+    SubLogical64Ext32,
+    Mul32,
+    Mul32Ext16,
+    Mul64,
+    Mul64Ext16,
+    Mul64Ext32,
+    And32,
+    And64,
+    Orr32,
+    Orr64,
+    Xor32,
+    Xor64,
+    AndNot32,
+    AndNot64,
+    OrrNot32,
+    OrrNot64,
+    XorNot32,
+    XorNot64,
+}
+
+/// Internal type UnaryOp: defined at src/isa/s390x/inst.isle line 732.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum UnaryOp {
+    Abs32,
+    Abs64,
+    Abs64Ext32,
+    Neg32,
+    Neg64,
+    Neg64Ext32,
+    PopcntByte,
+    PopcntReg,
+}
+
+/// Internal type ShiftOp: defined at src/isa/s390x/inst.isle line 745.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum ShiftOp {
+    RotL32,
+    RotL64,
+    LShL32,
+    LShL64,
+    LShR32,
+    LShR64,
+    AShR32,
+    AShR64,
+}
+
+/// Internal type CmpOp: defined at src/isa/s390x/inst.isle line 758.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum CmpOp {
+    CmpS32,
+    CmpS32Ext16,
+    CmpS64,
+    CmpS64Ext16,
+    CmpS64Ext32,
+    CmpL32,
+    CmpL32Ext16,
+    CmpL64,
+    CmpL64Ext16,
+    CmpL64Ext32,
+}
+
+/// Internal type FPUOp1: defined at src/isa/s390x/inst.isle line 773.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum FPUOp1 {
+    Abs32,
+    Abs64,
+    Neg32,
+    Neg64,
+    NegAbs32,
+    NegAbs64,
+    Sqrt32,
+    Sqrt64,
+    Cvt32To64,
+    Cvt64To32,
+}
+
+/// Internal type FPUOp2: defined at src/isa/s390x/inst.isle line 788.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum FPUOp2 {
+    Add32,
+    Add64,
+    Sub32,
+    Sub64,
+    Mul32,
+    Mul64,
+    Div32,
+    Div64,
+    Max32,
+    Max64,
+    Min32,
+    Min64,
+}
+
+/// Internal type FPUOp3: defined at src/isa/s390x/inst.isle line 805.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum FPUOp3 {
+    MAdd32,
+    MAdd64,
+    MSub32,
+    MSub64,
+}
+
+/// Internal type FpuToIntOp: defined at src/isa/s390x/inst.isle line 814.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum FpuToIntOp {
+    F32ToU32,
+    F32ToI32,
+    F32ToU64,
+    F32ToI64,
+    F64ToU32,
+    F64ToI32,
+    F64ToU64,
+    F64ToI64,
+}
+
+/// Internal type IntToFpuOp: defined at src/isa/s390x/inst.isle line 827.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum IntToFpuOp {
+    U32ToF32,
+    I32ToF32,
+    U32ToF64,
+    I32ToF64,
+    U64ToF32,
+    I64ToF32,
+    U64ToF64,
+    I64ToF64,
+}
+
+/// Internal type FpuRoundMode: defined at src/isa/s390x/inst.isle line 841.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum FpuRoundMode {
+    Minus32,
+    Minus64,
+    Plus32,
+    Plus64,
+    Zero32,
+    Zero64,
+    Nearest32,
+    Nearest64,
+}
+
+/// Internal type WritableRegPair: defined at src/isa/s390x/inst.isle line 1238.
+#[derive(Clone, Debug)]
+pub enum WritableRegPair {
+    WritableRegPair { hi: WritableReg, lo: WritableReg },
+}
+
+/// Internal type RegPair: defined at src/isa/s390x/inst.isle line 1260.
+#[derive(Clone, Debug)]
+pub enum RegPair {
+    RegPair { hi: Reg, lo: Reg },
+}
+
+/// Internal type ProducesBool: defined at src/isa/s390x/inst.isle line 2160.
+#[derive(Clone, Debug)]
+pub enum ProducesBool {
+    ProducesBool { producer: ProducesFlags, cond: Cond },
+}
+
+// Generated as internal constructor for term temp_reg.
+pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/prelude.isle line 70.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term lo_reg.
+pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/prelude.isle line 105.
+    let expr0_0 = C::put_in_regs(ctx, pattern0_0);
+    let expr1_0: usize = 0;
+    let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term value_regs_none.
+pub fn constructor_value_regs_none<C: Context>(
+    ctx: &mut C,
+    arg0: &SideEffectNoResult,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    if let &SideEffectNoResult::Inst {
+        inst: ref pattern1_0,
+    } = pattern0_0
+    {
+        // Rule at src/prelude.isle line 287.
+        let expr0_0 = C::emit(ctx, &pattern1_0);
+        let expr1_0 = C::value_regs_invalid(ctx);
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term with_flags.
+pub fn constructor_with_flags<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &ConsumesFlags,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        let pattern2_0 = arg1;
+        if let &ConsumesFlags::ConsumesFlags {
+            inst: ref pattern3_0,
+            result: pattern3_1,
+        } = pattern2_0
+        {
+            // Rule at src/prelude.isle line 308.
+            let expr0_0 = C::emit(ctx, &pattern1_0);
+            let expr1_0 = C::emit(ctx, &pattern3_0);
+            let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
+            return Some(expr2_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term with_flags_1.
+pub fn constructor_with_flags_1<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &ConsumesFlags,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        let pattern2_0 = arg1;
+        if let &ConsumesFlags::ConsumesFlags {
+            inst: ref pattern3_0,
+            result: pattern3_1,
+        } = pattern2_0
+        {
+            // Rule at src/prelude.isle line 316.
+            let expr0_0 = C::emit(ctx, &pattern1_0);
+            let expr1_0 = C::emit(ctx, &pattern3_0);
+            return Some(pattern3_1);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term with_flags_2.
+pub fn constructor_with_flags_2<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &ConsumesFlags,
+    arg2: &ConsumesFlags,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        let pattern2_0 = arg1;
+        if let &ConsumesFlags::ConsumesFlags {
+            inst: ref pattern3_0,
+            result: pattern3_1,
+        } = pattern2_0
+        {
+            let pattern4_0 = arg2;
+            if let &ConsumesFlags::ConsumesFlags {
+                inst: ref pattern5_0,
+                result: pattern5_1,
+            } = pattern4_0
+            {
+                // Rule at src/prelude.isle line 326.
+                let expr0_0 = C::emit(ctx, &pattern1_0);
+                let expr1_0 = C::emit(ctx, &pattern5_0);
+                let expr2_0 = C::emit(ctx, &pattern3_0);
+                let expr3_0 = C::value_regs(ctx, pattern3_1, pattern5_1);
+                return Some(expr3_0);
+            }
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term mask_amt_reg.
+pub fn constructor_mask_amt_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 1016.
+        let expr0_0: i64 = -1;
+        let expr1_0 = C::mask_amt_imm(ctx, pattern1_0, expr0_0);
+        let expr2_0 = C::u8_as_u16(ctx, expr1_0);
+        let expr3_0: u8 = 0;
+        let expr4_0 = C::uimm16shifted(ctx, expr2_0, expr3_0);
+        let expr5_0 = constructor_and_uimm16shifted(ctx, pattern1_0, pattern2_0, expr4_0)?;
+        return Some(expr5_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 1019.
+        return Some(pattern2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term lower_address.
+pub fn constructor_lower_address<C: Context>(
+    ctx: &mut C,
+    arg0: MemFlags,
+    arg1: Value,
+    arg2: Offset32,
+) -> Option<MemArg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if let Some(pattern2_0) = C::def_inst(ctx, pattern1_0) {
+        let pattern3_0 = C::inst_data(ctx, pattern2_0);
+        if let &InstructionData::Binary {
+            opcode: ref pattern4_0,
+            args: ref pattern4_1,
+        } = &pattern3_0
+        {
+            if let &Opcode::Iadd = &pattern4_0 {
+                let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                let pattern7_0 = arg2;
+                let pattern8_0 = C::i64_from_offset(ctx, pattern7_0);
+                if pattern8_0 == 0 {
+                    // Rule at src/isa/s390x/inst.isle line 1102.
+                    let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                    let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                    let expr2_0 = C::memarg_reg_plus_reg(ctx, expr0_0, expr1_0, pattern0_0);
+                    return Some(expr2_0);
+                }
+            }
+        }
+        if let Some((pattern3_0, pattern3_1, pattern3_2)) = C::symbol_value_data(ctx, pattern2_0) {
+            if let Some(()) = C::reloc_distance_near(ctx, &pattern3_1) {
+                let pattern5_0 = arg2;
+                let pattern6_0 = C::i64_from_offset(ctx, pattern5_0);
+                let closure7 = || {
+                    return Some(pattern3_2);
+                };
+                if let Some(pattern7_0) = closure7() {
+                    if let Some(pattern8_0) =
+                        C::memarg_symbol_offset_sum(ctx, pattern6_0, pattern7_0)
+                    {
+                        // Rule at src/isa/s390x/inst.isle line 1105.
+                        let expr0_0 = C::memarg_symbol(ctx, pattern3_0, pattern8_0, pattern0_0);
+                        return Some(expr0_0);
+                    }
+                }
+            }
+        }
+    }
+    let pattern2_0 = arg2;
+    let pattern3_0 = C::i64_from_offset(ctx, pattern2_0);
+    // Rule at src/isa/s390x/inst.isle line 1099.
+    let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+    let expr1_0 = C::memarg_reg_plus_off(ctx, expr0_0, pattern3_0, pattern0_0);
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term stack_addr_impl.
+pub fn constructor_stack_addr_impl<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: StackSlot,
+    arg2: Offset32,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1132.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = C::abi_stackslot_addr(ctx, expr0_0, pattern1_0, pattern2_0);
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term sink_load.
+pub fn constructor_sink_load<C: Context>(ctx: &mut C, arg0: Inst) -> Option<MemArg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::inst_data(ctx, pattern0_0);
+    if let &InstructionData::Load {
+        opcode: ref pattern2_0,
+        arg: pattern2_1,
+        flags: pattern2_2,
+        offset: pattern2_3,
+    } = &pattern1_0
+    {
+        if let &Opcode::Load = &pattern2_0 {
+            // Rule at src/isa/s390x/inst.isle line 1202.
+            let expr0_0 = C::sink_inst(ctx, pattern0_0);
+            let expr1_0 = constructor_lower_address(ctx, pattern2_2, pattern2_1, pattern2_3)?;
+            return Some(expr1_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sink_sload16.
+pub fn constructor_sink_sload16<C: Context>(ctx: &mut C, arg0: Inst) -> Option<MemArg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::inst_data(ctx, pattern0_0);
+    if let &InstructionData::Load {
+        opcode: ref pattern2_0,
+        arg: pattern2_1,
+        flags: pattern2_2,
+        offset: pattern2_3,
+    } = &pattern1_0
+    {
+        if let &Opcode::Sload16 = &pattern2_0 {
+            // Rule at src/isa/s390x/inst.isle line 1209.
+            let expr0_0 = C::sink_inst(ctx, pattern0_0);
+            let expr1_0 = constructor_lower_address(ctx, pattern2_2, pattern2_1, pattern2_3)?;
+            return Some(expr1_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sink_sload32.
+pub fn constructor_sink_sload32<C: Context>(ctx: &mut C, arg0: Inst) -> Option<MemArg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::inst_data(ctx, pattern0_0);
+    if let &InstructionData::Load {
+        opcode: ref pattern2_0,
+        arg: pattern2_1,
+        flags: pattern2_2,
+        offset: pattern2_3,
+    } = &pattern1_0
+    {
+        if let &Opcode::Sload32 = &pattern2_0 {
+            // Rule at src/isa/s390x/inst.isle line 1216.
+            let expr0_0 = C::sink_inst(ctx, pattern0_0);
+            let expr1_0 = constructor_lower_address(ctx, pattern2_2, pattern2_1, pattern2_3)?;
+            return Some(expr1_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sink_uload16.
+pub fn constructor_sink_uload16<C: Context>(ctx: &mut C, arg0: Inst) -> Option<MemArg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::inst_data(ctx, pattern0_0);
+    if let &InstructionData::Load {
+        opcode: ref pattern2_0,
+        arg: pattern2_1,
+        flags: pattern2_2,
+        offset: pattern2_3,
+    } = &pattern1_0
+    {
+        if let &Opcode::Uload16 = &pattern2_0 {
+            // Rule at src/isa/s390x/inst.isle line 1223.
+            let expr0_0 = C::sink_inst(ctx, pattern0_0);
+            let expr1_0 = constructor_lower_address(ctx, pattern2_2, pattern2_1, pattern2_3)?;
+            return Some(expr1_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sink_uload32.
+pub fn constructor_sink_uload32<C: Context>(ctx: &mut C, arg0: Inst) -> Option<MemArg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::inst_data(ctx, pattern0_0);
+    if let &InstructionData::Load {
+        opcode: ref pattern2_0,
+        arg: pattern2_1,
+        flags: pattern2_2,
+        offset: pattern2_3,
+    } = &pattern1_0
+    {
+        if let &Opcode::Uload32 = &pattern2_0 {
+            // Rule at src/isa/s390x/inst.isle line 1230.
+            let expr0_0 = C::sink_inst(ctx, pattern0_0);
+            let expr1_0 = constructor_lower_address(ctx, pattern2_2, pattern2_1, pattern2_3)?;
+            return Some(expr1_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term temp_writable_regpair.
+pub fn constructor_temp_writable_regpair<C: Context>(ctx: &mut C) -> Option<WritableRegPair> {
+    // Rule at src/isa/s390x/inst.isle line 1243.
+    let expr0_0: u8 = 0;
+    let expr1_0 = C::writable_gpr(ctx, expr0_0);
+    let expr2_0: u8 = 1;
+    let expr3_0 = C::writable_gpr(ctx, expr2_0);
+    let expr4_0 = WritableRegPair::WritableRegPair {
+        hi: expr1_0,
+        lo: expr3_0,
+    };
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term copy_writable_regpair.
+pub fn constructor_copy_writable_regpair<C: Context>(
+    ctx: &mut C,
+    arg0: &RegPair,
+) -> Option<WritableRegPair> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1249.
+    let expr0_0 = constructor_temp_writable_regpair(ctx)?;
+    return Some(expr0_0);
+}
+
+// Generated as internal constructor for term writable_regpair_hi.
+pub fn constructor_writable_regpair_hi<C: Context>(
+    ctx: &mut C,
+    arg0: &WritableRegPair,
+) -> Option<WritableReg> {
+    let pattern0_0 = arg0;
+    if let &WritableRegPair::WritableRegPair {
+        hi: pattern1_0,
+        lo: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 1253.
+        return Some(pattern1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term writable_regpair_lo.
+pub fn constructor_writable_regpair_lo<C: Context>(
+    ctx: &mut C,
+    arg0: &WritableRegPair,
+) -> Option<WritableReg> {
+    let pattern0_0 = arg0;
+    if let &WritableRegPair::WritableRegPair {
+        hi: pattern1_0,
+        lo: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 1257.
+        return Some(pattern1_1);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term writable_regpair_to_regpair.
+pub fn constructor_writable_regpair_to_regpair<C: Context>(
+    ctx: &mut C,
+    arg0: &WritableRegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    if let &WritableRegPair::WritableRegPair {
+        hi: pattern1_0,
+        lo: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 1264.
+        let expr0_0 = C::writable_reg_to_reg(ctx, pattern1_0);
+        let expr1_0 = C::writable_reg_to_reg(ctx, pattern1_1);
+        let expr2_0 = RegPair::RegPair {
+            hi: expr0_0,
+            lo: expr1_0,
+        };
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term uninitialized_regpair.
+pub fn constructor_uninitialized_regpair<C: Context>(ctx: &mut C) -> Option<RegPair> {
+    // Rule at src/isa/s390x/inst.isle line 1269.
+    let expr0_0 = constructor_temp_writable_regpair(ctx)?;
+    let expr1_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term regpair_hi.
+pub fn constructor_regpair_hi<C: Context>(ctx: &mut C, arg0: &RegPair) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let &RegPair::RegPair {
+        hi: pattern1_0,
+        lo: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 1274.
+        return Some(pattern1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term regpair_lo.
+pub fn constructor_regpair_lo<C: Context>(ctx: &mut C, arg0: &RegPair) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let &RegPair::RegPair {
+        hi: pattern1_0,
+        lo: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 1278.
+        return Some(pattern1_1);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term alu_rrr.
+pub fn constructor_alu_rrr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1292.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::AluRRR {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+        rm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_rrsimm16.
+pub fn constructor_alu_rrsimm16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: i16,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1299.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::AluRRSImm16 {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+        imm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_rr.
+pub fn constructor_alu_rr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1306.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::AluRR {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_rx.
+pub fn constructor_alu_rx<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1313.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::AluRX {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        mem: pattern3_0.clone(),
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_rsimm16.
+pub fn constructor_alu_rsimm16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: i16,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1320.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::AluRSImm16 {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        imm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_rsimm32.
+pub fn constructor_alu_rsimm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: i32,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1327.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::AluRSImm32 {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        imm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_ruimm32.
+pub fn constructor_alu_ruimm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: u32,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1334.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::AluRUImm32 {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        imm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_ruimm16shifted.
+pub fn constructor_alu_ruimm16shifted<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: UImm16Shifted,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1341.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::AluRUImm16Shifted {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        imm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term alu_ruimm32shifted.
+pub fn constructor_alu_ruimm32shifted<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: UImm32Shifted,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1348.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::AluRUImm32Shifted {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        imm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term smul_wide.
+pub fn constructor_smul_wide<C: Context>(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1355.
+    let expr0_0 = constructor_temp_writable_regpair(ctx)?;
+    let expr1_0 = MInst::SMulWide {
+        rn: pattern0_0,
+        rm: pattern1_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term umul_wide.
+pub fn constructor_umul_wide<C: Context>(ctx: &mut C, arg0: Reg, arg1: Reg) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1362.
+    let expr0_0 = constructor_temp_writable_regpair(ctx)?;
+    let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?;
+    let expr2_0 = MInst::Mov64 {
+        rd: expr1_0,
+        rm: pattern1_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = MInst::UMulWide { rn: pattern0_0 };
+    let expr5_0 = C::emit(ctx, &expr4_0);
+    let expr6_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr6_0);
+}
+
+// Generated as internal constructor for term sdivmod32.
+pub fn constructor_sdivmod32<C: Context>(
+    ctx: &mut C,
+    arg0: &RegPair,
+    arg1: Reg,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1370.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?;
+    let expr1_0 = MInst::SDivMod32 { rn: pattern1_0 };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term sdivmod64.
+pub fn constructor_sdivmod64<C: Context>(
+    ctx: &mut C,
+    arg0: &RegPair,
+    arg1: Reg,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1377.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?;
+    let expr1_0 = MInst::SDivMod64 { rn: pattern1_0 };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term udivmod32.
+pub fn constructor_udivmod32<C: Context>(
+    ctx: &mut C,
+    arg0: &RegPair,
+    arg1: Reg,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1384.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?;
+    let expr1_0 = MInst::UDivMod32 { rn: pattern1_0 };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term udivmod64.
+pub fn constructor_udivmod64<C: Context>(
+    ctx: &mut C,
+    arg0: &RegPair,
+    arg1: Reg,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1391.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern0_0)?;
+    let expr1_0 = MInst::UDivMod64 { rn: pattern1_0 };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term shift_rr.
+pub fn constructor_shift_rr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ShiftOp,
+    arg2: Reg,
+    arg3: u8,
+    arg4: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 1398.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::ShiftRR {
+        shift_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+        shift_imm: pattern3_0,
+        shift_reg: pattern4_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term unary_rr.
+pub fn constructor_unary_rr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &UnaryOp,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1405.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::UnaryRR {
+        op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term cmp_rr.
+pub fn constructor_cmp_rr<C: Context>(
+    ctx: &mut C,
+    arg0: &CmpOp,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1412.
+    let expr0_0 = MInst::CmpRR {
+        op: pattern0_0.clone(),
+        rn: pattern1_0,
+        rm: pattern2_0,
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term cmp_rx.
+pub fn constructor_cmp_rx<C: Context>(
+    ctx: &mut C,
+    arg0: &CmpOp,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1418.
+    let expr0_0 = MInst::CmpRX {
+        op: pattern0_0.clone(),
+        rn: pattern1_0,
+        mem: pattern2_0.clone(),
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term cmp_rsimm16.
+pub fn constructor_cmp_rsimm16<C: Context>(
+    ctx: &mut C,
+    arg0: &CmpOp,
+    arg1: Reg,
+    arg2: i16,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1424.
+    let expr0_0 = MInst::CmpRSImm16 {
+        op: pattern0_0.clone(),
+        rn: pattern1_0,
+        imm: pattern2_0,
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term cmp_rsimm32.
+pub fn constructor_cmp_rsimm32<C: Context>(
+    ctx: &mut C,
+    arg0: &CmpOp,
+    arg1: Reg,
+    arg2: i32,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1430.
+    let expr0_0 = MInst::CmpRSImm32 {
+        op: pattern0_0.clone(),
+        rn: pattern1_0,
+        imm: pattern2_0,
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term cmp_ruimm32.
+pub fn constructor_cmp_ruimm32<C: Context>(
+    ctx: &mut C,
+    arg0: &CmpOp,
+    arg1: Reg,
+    arg2: u32,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1436.
+    let expr0_0 = MInst::CmpRUImm32 {
+        op: pattern0_0.clone(),
+        rn: pattern1_0,
+        imm: pattern2_0,
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term atomic_rmw_impl.
+pub fn constructor_atomic_rmw_impl<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ALUOp,
+    arg2: Reg,
+    arg3: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1442.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::AtomicRmw {
+        alu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+        mem: pattern3_0.clone(),
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term atomic_cas32.
+pub fn constructor_atomic_cas32<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1449.
+    let expr0_0: Type = I32;
+    let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?;
+    let expr2_0 = MInst::AtomicCas32 {
+        rd: expr1_0,
+        rn: pattern1_0,
+        mem: pattern2_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term atomic_cas64.
+pub fn constructor_atomic_cas64<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1456.
+    let expr0_0: Type = I64;
+    let expr1_0 = constructor_copy_writable_reg(ctx, expr0_0, pattern0_0)?;
+    let expr2_0 = MInst::AtomicCas64 {
+        rd: expr1_0,
+        rn: pattern1_0,
+        mem: pattern2_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term fence_impl.
+pub fn constructor_fence_impl<C: Context>(ctx: &mut C) -> Option<SideEffectNoResult> {
+    // Rule at src/isa/s390x/inst.isle line 1463.
+    let expr0_0 = MInst::Fence;
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term load32.
+pub fn constructor_load32<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1468.
+    let expr0_0: Type = I32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::Load32 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term load64.
+pub fn constructor_load64<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1475.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::Load64 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term loadrev16.
+pub fn constructor_loadrev16<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1482.
+    let expr0_0: Type = I32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::LoadRev16 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term loadrev32.
+pub fn constructor_loadrev32<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1489.
+    let expr0_0: Type = I32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::LoadRev32 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term loadrev64.
+pub fn constructor_loadrev64<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1496.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::LoadRev64 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term store8.
+pub fn constructor_store8<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1503.
+    let expr0_0 = MInst::Store8 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term store16.
+pub fn constructor_store16<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1508.
+    let expr0_0 = MInst::Store16 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term store32.
+pub fn constructor_store32<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1513.
+    let expr0_0 = MInst::Store32 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term store64.
+pub fn constructor_store64<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1518.
+    let expr0_0 = MInst::Store64 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term store8_imm.
+pub fn constructor_store8_imm<C: Context>(
+    ctx: &mut C,
+    arg0: u8,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1523.
+    let expr0_0 = MInst::StoreImm8 {
+        imm: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term store16_imm.
+pub fn constructor_store16_imm<C: Context>(
+    ctx: &mut C,
+    arg0: i16,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1528.
+    let expr0_0 = MInst::StoreImm16 {
+        imm: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term store32_simm16.
+pub fn constructor_store32_simm16<C: Context>(
+    ctx: &mut C,
+    arg0: i16,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1533.
+    let expr0_0 = MInst::StoreImm32SExt16 {
+        imm: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term store64_simm16.
+pub fn constructor_store64_simm16<C: Context>(
+    ctx: &mut C,
+    arg0: i16,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1538.
+    let expr0_0 = MInst::StoreImm64SExt16 {
+        imm: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term storerev16.
+pub fn constructor_storerev16<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1543.
+    let expr0_0 = MInst::StoreRev16 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term storerev32.
+pub fn constructor_storerev32<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1548.
+    let expr0_0 = MInst::StoreRev32 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term storerev64.
+pub fn constructor_storerev64<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1553.
+    let expr0_0 = MInst::StoreRev64 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpu_rr.
+pub fn constructor_fpu_rr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &FPUOp1,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1558.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::FpuRR {
+        fpu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term fpu_rrr.
+pub fn constructor_fpu_rrr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &FPUOp2,
+    arg2: Reg,
+    arg3: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1565.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::FpuRRR {
+        fpu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term fpu_rrrr.
+pub fn constructor_fpu_rrrr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &FPUOp3,
+    arg2: Reg,
+    arg3: Reg,
+    arg4: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 1572.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern2_0)?;
+    let expr1_0 = MInst::FpuRRRR {
+        fpu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern3_0,
+        rm: pattern4_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term fpu_copysign.
+pub fn constructor_fpu_copysign<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1579.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::FpuCopysign {
+        rd: expr0_0,
+        rn: pattern1_0,
+        rm: pattern2_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term fpu_cmp32.
+pub fn constructor_fpu_cmp32<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1586.
+    let expr0_0 = MInst::FpuCmp32 {
+        rn: pattern0_0,
+        rm: pattern1_0,
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term fpu_cmp64.
+pub fn constructor_fpu_cmp64<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1592.
+    let expr0_0 = MInst::FpuCmp64 {
+        rn: pattern0_0,
+        rm: pattern1_0,
+    };
+    let expr1_0 = C::invalid_reg(ctx);
+    let expr2_0 = ProducesFlags::ProducesFlags {
+        inst: expr0_0,
+        result: expr1_0,
+    };
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term fpu_to_int.
+pub fn constructor_fpu_to_int<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &FpuToIntOp,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1598.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::FpuToInt {
+        op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+    };
+    let expr2_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    let expr3_0 = ProducesFlags::ProducesFlags {
+        inst: expr1_0,
+        result: expr2_0,
+    };
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term int_to_fpu.
+pub fn constructor_int_to_fpu<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &IntToFpuOp,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1605.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::IntToFpu {
+        op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term fpu_round.
+pub fn constructor_fpu_round<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &FpuRoundMode,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1612.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::FpuRound {
+        op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term fpuvec_rrr.
+pub fn constructor_fpuvec_rrr<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &FPUOp2,
+    arg2: Reg,
+    arg3: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 1619.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = MInst::FpuVecRRR {
+        fpu_op: pattern1_0.clone(),
+        rd: expr0_0,
+        rn: pattern2_0,
+        rm: pattern3_0,
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term mov_to_fpr.
+pub fn constructor_mov_to_fpr<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1626.
+    let expr0_0: Type = F64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::MovToFpr {
+        rd: expr1_0,
+        rn: pattern0_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term mov_from_fpr.
+pub fn constructor_mov_from_fpr<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1633.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::MovFromFpr {
+        rd: expr1_0,
+        rn: pattern0_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term fpu_load32.
+pub fn constructor_fpu_load32<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1640.
+    let expr0_0: Type = F32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::FpuLoad32 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term fpu_load64.
+pub fn constructor_fpu_load64<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1647.
+    let expr0_0: Type = F64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::FpuLoad64 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term fpu_loadrev32.
+pub fn constructor_fpu_loadrev32<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1654.
+    let expr0_0: Type = F32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::FpuLoadRev32 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term fpu_loadrev64.
+pub fn constructor_fpu_loadrev64<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1661.
+    let expr0_0: Type = F64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::FpuLoadRev64 {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term fpu_store32.
+pub fn constructor_fpu_store32<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1668.
+    let expr0_0 = MInst::FpuStore32 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpu_store64.
+pub fn constructor_fpu_store64<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1673.
+    let expr0_0 = MInst::FpuStore64 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpu_storerev32.
+pub fn constructor_fpu_storerev32<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1678.
+    let expr0_0 = MInst::FpuStoreRev32 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpu_storerev64.
+pub fn constructor_fpu_storerev64<C: Context>(
+    ctx: &mut C,
+    arg0: Reg,
+    arg1: &MemArg,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1683.
+    let expr0_0 = MInst::FpuStoreRev64 {
+        rd: pattern0_0,
+        mem: pattern1_0.clone(),
+    };
+    let expr1_0 = SideEffectNoResult::Inst { inst: expr0_0 };
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term load_ext_name_far.
+pub fn constructor_load_ext_name_far<C: Context>(
+    ctx: &mut C,
+    arg0: BoxExternalName,
+    arg1: i64,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1688.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::LoadExtNameFar {
+        rd: expr1_0,
+        name: pattern0_0,
+        offset: pattern1_0,
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term load_addr.
+pub fn constructor_load_addr<C: Context>(ctx: &mut C, arg0: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 1695.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = MInst::LoadAddr {
+        rd: expr1_0,
+        mem: pattern0_0.clone(),
+    };
+    let expr3_0 = C::emit(ctx, &expr2_0);
+    let expr4_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term drop_flags.
+pub fn constructor_drop_flags<C: Context>(ctx: &mut C, arg0: &ProducesFlags) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 1702.
+        let expr0_0 = C::emit(ctx, &pattern1_0);
+        return Some(pattern1_1);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_mov.
+pub fn constructor_emit_mov<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: WritableReg,
+    arg2: Reg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1718.
+        let expr0_0 = MInst::FpuMove32 {
+            rd: pattern2_0,
+            rn: pattern3_0,
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if pattern0_0 == F64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1721.
+        let expr0_0 = MInst::FpuMove64 {
+            rd: pattern2_0,
+            rn: pattern3_0,
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1712.
+        let expr0_0 = MInst::Mov32 {
+            rd: pattern2_0,
+            rm: pattern3_0,
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1715.
+        let expr0_0 = MInst::Mov64 {
+            rd: pattern2_0,
+            rm: pattern3_0,
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term copy_writable_reg.
+pub fn constructor_copy_writable_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+) -> Option<WritableReg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1726.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern1_0)?;
+    return Some(expr0_0);
+}
+
+// Generated as internal constructor for term copy_reg.
+pub fn constructor_copy_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1733.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern1_0)?;
+    let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term emit_imm.
+pub fn constructor_emit_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: WritableReg,
+    arg2: u64,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1788.
+        let expr0_0 = C::u64_as_u32(ctx, pattern3_0);
+        let expr1_0 = MInst::LoadFpuConst32 {
+            rd: pattern2_0,
+            const_data: expr0_0,
+        };
+        let expr2_0 = C::emit(ctx, &expr1_0);
+        return Some(expr2_0);
+    }
+    if pattern0_0 == F64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1793.
+        let expr0_0 = MInst::LoadFpuConst64 {
+            rd: pattern2_0,
+            const_data: pattern3_0,
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if let Some(pattern1_0) = C::fits_in_16(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1742.
+        let expr0_0 = C::u64_as_i16(ctx, pattern3_0);
+        let expr1_0 = MInst::Mov32SImm16 {
+            rd: pattern2_0,
+            imm: expr0_0,
+        };
+        let expr2_0 = C::emit(ctx, &expr1_0);
+        return Some(expr2_0);
+    }
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) {
+            // Rule at src/isa/s390x/inst.isle line 1746.
+            let expr0_0 = MInst::Mov32SImm16 {
+                rd: pattern2_0,
+                imm: pattern4_0,
+            };
+            let expr1_0 = C::emit(ctx, &expr0_0);
+            return Some(expr1_0);
+        }
+        // Rule at src/isa/s390x/inst.isle line 1750.
+        let expr0_0 = C::u64_as_u32(ctx, pattern3_0);
+        let expr1_0 = MInst::Mov32Imm {
+            rd: pattern2_0,
+            imm: expr0_0,
+        };
+        let expr2_0 = C::emit(ctx, &expr1_0);
+        return Some(expr2_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        if let Some(pattern4_0) = C::u64_nonzero_hipart(ctx, pattern3_0) {
+            if let Some(pattern5_0) = C::u64_nonzero_lopart(ctx, pattern3_0) {
+                // Rule at src/isa/s390x/inst.isle line 1770.
+                let expr0_0 = constructor_emit_imm(ctx, pattern1_0, pattern2_0, pattern4_0)?;
+                let expr1_0 = constructor_emit_insert_imm(ctx, pattern2_0, pattern5_0)?;
+                return Some(expr1_0);
+            }
+        }
+        if let Some(pattern4_0) = C::i16_from_u64(ctx, pattern3_0) {
+            // Rule at src/isa/s390x/inst.isle line 1754.
+            let expr0_0 = MInst::Mov64SImm16 {
+                rd: pattern2_0,
+                imm: pattern4_0,
+            };
+            let expr1_0 = C::emit(ctx, &expr0_0);
+            return Some(expr1_0);
+        }
+        if let Some(pattern4_0) = C::i32_from_u64(ctx, pattern3_0) {
+            // Rule at src/isa/s390x/inst.isle line 1758.
+            let expr0_0 = MInst::Mov64SImm32 {
+                rd: pattern2_0,
+                imm: pattern4_0,
+            };
+            let expr1_0 = C::emit(ctx, &expr0_0);
+            return Some(expr1_0);
+        }
+        if let Some(pattern4_0) = C::uimm32shifted_from_u64(ctx, pattern3_0) {
+            // Rule at src/isa/s390x/inst.isle line 1766.
+            let expr0_0 = MInst::Mov64UImm32Shifted {
+                rd: pattern2_0,
+                imm: pattern4_0,
+            };
+            let expr1_0 = C::emit(ctx, &expr0_0);
+            return Some(expr1_0);
+        }
+        if let Some(pattern4_0) = C::uimm16shifted_from_u64(ctx, pattern3_0) {
+            // Rule at src/isa/s390x/inst.isle line 1762.
+            let expr0_0 = MInst::Mov64UImm16Shifted {
+                rd: pattern2_0,
+                imm: pattern4_0,
+            };
+            let expr1_0 = C::emit(ctx, &expr0_0);
+            return Some(expr1_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_insert_imm.
+pub fn constructor_emit_insert_imm<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: u64,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if let Some(pattern2_0) = C::uimm32shifted_from_u64(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 1783.
+        let expr0_0 = MInst::Insert64UImm32Shifted {
+            rd: pattern0_0,
+            imm: pattern2_0,
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::uimm16shifted_from_u64(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 1779.
+        let expr0_0 = MInst::Insert64UImm16Shifted {
+            rd: pattern0_0,
+            imm: pattern2_0,
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term imm.
+pub fn constructor_imm<C: Context>(ctx: &mut C, arg0: Type, arg1: u64) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1798.
+    let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+    let expr1_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern1_0)?;
+    let expr2_0 = C::writable_reg_to_reg(ctx, expr0_0);
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term imm_regpair_lo.
+pub fn constructor_imm_regpair_lo<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: u64,
+    arg2: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1806.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?;
+    let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?;
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term imm_regpair_hi.
+pub fn constructor_imm_regpair_hi<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: u64,
+    arg2: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1814.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern2_0)?;
+    let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr1_0, pattern1_0)?;
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term ty_ext32.
+pub fn constructor_ty_ext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<Type> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/inst.isle line 1824.
+        let expr0_0: Type = I32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 1825.
+        let expr0_0: Type = I32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 1826.
+        let expr0_0: Type = I32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 1827.
+        let expr0_0: Type = I64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term ty_ext64.
+pub fn constructor_ty_ext64<C: Context>(ctx: &mut C, arg0: Type) -> Option<Type> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/inst.isle line 1831.
+        let expr0_0: Type = I64;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 1832.
+        let expr0_0: Type = I64;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 1833.
+        let expr0_0: Type = I64;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 1834.
+        let expr0_0: Type = I64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_zext32_reg.
+pub fn constructor_emit_zext32_reg<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1839.
+    let expr0_0: bool = false;
+    let expr1_0 = C::ty_bits(ctx, pattern1_0);
+    let expr2_0: u8 = 32;
+    let expr3_0 = MInst::Extend {
+        rd: pattern0_0,
+        rn: pattern2_0,
+        signed: expr0_0,
+        from_bits: expr1_0,
+        to_bits: expr2_0,
+    };
+    let expr4_0 = C::emit(ctx, &expr3_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term emit_sext32_reg.
+pub fn constructor_emit_sext32_reg<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1845.
+    let expr0_0: bool = true;
+    let expr1_0 = C::ty_bits(ctx, pattern1_0);
+    let expr2_0: u8 = 32;
+    let expr3_0 = MInst::Extend {
+        rd: pattern0_0,
+        rn: pattern2_0,
+        signed: expr0_0,
+        from_bits: expr1_0,
+        to_bits: expr2_0,
+    };
+    let expr4_0 = C::emit(ctx, &expr3_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term emit_zext64_reg.
+pub fn constructor_emit_zext64_reg<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1851.
+    let expr0_0: bool = false;
+    let expr1_0 = C::ty_bits(ctx, pattern1_0);
+    let expr2_0: u8 = 64;
+    let expr3_0 = MInst::Extend {
+        rd: pattern0_0,
+        rn: pattern2_0,
+        signed: expr0_0,
+        from_bits: expr1_0,
+        to_bits: expr2_0,
+    };
+    let expr4_0 = C::emit(ctx, &expr3_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term emit_sext64_reg.
+pub fn constructor_emit_sext64_reg<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 1857.
+    let expr0_0: bool = true;
+    let expr1_0 = C::ty_bits(ctx, pattern1_0);
+    let expr2_0: u8 = 64;
+    let expr3_0 = MInst::Extend {
+        rd: pattern0_0,
+        rn: pattern2_0,
+        signed: expr0_0,
+        from_bits: expr1_0,
+        to_bits: expr2_0,
+    };
+    let expr4_0 = C::emit(ctx, &expr3_0);
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term zext32_reg.
+pub fn constructor_zext32_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1863.
+    let expr0_0: Type = I32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_zext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term sext32_reg.
+pub fn constructor_sext32_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1871.
+    let expr0_0: Type = I32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_sext32_reg(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term zext64_reg.
+pub fn constructor_zext64_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1879.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_zext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term sext64_reg.
+pub fn constructor_sext64_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1887.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_sext64_reg(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term emit_zext32_mem.
+pub fn constructor_emit_zext32_mem<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: &MemArg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if pattern1_0 == I8 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1895.
+        let expr0_0 = MInst::Load32ZExt8 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if pattern1_0 == I16 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1896.
+        let expr0_0 = MInst::Load32ZExt16 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_sext32_mem.
+pub fn constructor_emit_sext32_mem<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: &MemArg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if pattern1_0 == I8 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1900.
+        let expr0_0 = MInst::Load32SExt8 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if pattern1_0 == I16 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1901.
+        let expr0_0 = MInst::Load32SExt16 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_zext64_mem.
+pub fn constructor_emit_zext64_mem<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: &MemArg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if pattern1_0 == I8 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1905.
+        let expr0_0 = MInst::Load64ZExt8 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if pattern1_0 == I16 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1906.
+        let expr0_0 = MInst::Load64ZExt16 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if pattern1_0 == I32 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1907.
+        let expr0_0 = MInst::Load64ZExt32 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_sext64_mem.
+pub fn constructor_emit_sext64_mem<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Type,
+    arg2: &MemArg,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if pattern1_0 == I8 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1911.
+        let expr0_0 = MInst::Load64SExt8 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if pattern1_0 == I16 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1912.
+        let expr0_0 = MInst::Load64SExt16 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    if pattern1_0 == I32 {
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 1913.
+        let expr0_0 = MInst::Load64SExt32 {
+            rd: pattern0_0,
+            mem: pattern3_0.clone(),
+        };
+        let expr1_0 = C::emit(ctx, &expr0_0);
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term zext32_mem.
+pub fn constructor_zext32_mem<C: Context>(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1917.
+    let expr0_0: Type = I32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_zext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term sext32_mem.
+pub fn constructor_sext32_mem<C: Context>(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1924.
+    let expr0_0: Type = I32;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_sext32_mem(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term zext64_mem.
+pub fn constructor_zext64_mem<C: Context>(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1931.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_zext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term sext64_mem.
+pub fn constructor_sext64_mem<C: Context>(ctx: &mut C, arg0: Type, arg1: &MemArg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 1938.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = constructor_emit_sext64_mem(ctx, expr1_0, pattern0_0, pattern1_0)?;
+    let expr3_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term emit_put_in_reg_zext32.
+pub fn constructor_emit_put_in_reg_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Value,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = C::value_type(ctx, pattern1_0);
+    if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 1946.
+        let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?;
+        let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::fits_in_16(ctx, pattern2_0) {
+        if let Some(pattern4_0) = C::sinkable_inst(ctx, pattern1_0) {
+            let pattern5_0 = C::inst_data(ctx, pattern4_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern6_0,
+                arg: pattern6_1,
+                flags: pattern6_2,
+                offset: pattern6_3,
+            } = &pattern5_0
+            {
+                if let &Opcode::Load = &pattern6_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern6_2) {
+                        // Rule at src/isa/s390x/inst.isle line 1948.
+                        let expr0_0 = constructor_sink_load(ctx, pattern4_0)?;
+                        let expr1_0 =
+                            constructor_emit_zext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 1950.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_zext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) {
+        // Rule at src/isa/s390x/inst.isle line 1952.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_put_in_reg_sext32.
+pub fn constructor_emit_put_in_reg_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Value,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = C::value_type(ctx, pattern1_0);
+    if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 1957.
+        let expr0_0 = constructor_ty_ext32(ctx, pattern2_0)?;
+        let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::fits_in_16(ctx, pattern2_0) {
+        if let Some(pattern4_0) = C::sinkable_inst(ctx, pattern1_0) {
+            let pattern5_0 = C::inst_data(ctx, pattern4_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern6_0,
+                arg: pattern6_1,
+                flags: pattern6_2,
+                offset: pattern6_3,
+            } = &pattern5_0
+            {
+                if let &Opcode::Load = &pattern6_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern6_2) {
+                        // Rule at src/isa/s390x/inst.isle line 1959.
+                        let expr0_0 = constructor_sink_load(ctx, pattern4_0)?;
+                        let expr1_0 =
+                            constructor_emit_sext32_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 1961.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_sext32_reg(ctx, pattern0_0, pattern3_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) {
+        // Rule at src/isa/s390x/inst.isle line 1963.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_put_in_reg_zext64.
+pub fn constructor_emit_put_in_reg_zext64<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Value,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = C::value_type(ctx, pattern1_0);
+    if let Some(pattern3_0) = C::u64_from_value(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 1968.
+        let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?;
+        let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::gpr32_ty(ctx, pattern2_0) {
+        if let Some(pattern4_0) = C::sinkable_inst(ctx, pattern1_0) {
+            let pattern5_0 = C::inst_data(ctx, pattern4_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern6_0,
+                arg: pattern6_1,
+                flags: pattern6_2,
+                offset: pattern6_3,
+            } = &pattern5_0
+            {
+                if let &Opcode::Load = &pattern6_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern6_2) {
+                        // Rule at src/isa/s390x/inst.isle line 1970.
+                        let expr0_0 = constructor_sink_load(ctx, pattern4_0)?;
+                        let expr1_0 =
+                            constructor_emit_zext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 1972.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_zext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) {
+        // Rule at src/isa/s390x/inst.isle line 1974.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_put_in_reg_sext64.
+pub fn constructor_emit_put_in_reg_sext64<C: Context>(
+    ctx: &mut C,
+    arg0: WritableReg,
+    arg1: Value,
+) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = C::value_type(ctx, pattern1_0);
+    if let Some(pattern3_0) = C::u64_from_signed_value(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 1979.
+        let expr0_0 = constructor_ty_ext64(ctx, pattern2_0)?;
+        let expr1_0 = constructor_emit_imm(ctx, expr0_0, pattern0_0, pattern3_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::gpr32_ty(ctx, pattern2_0) {
+        if let Some(pattern4_0) = C::sinkable_inst(ctx, pattern1_0) {
+            let pattern5_0 = C::inst_data(ctx, pattern4_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern6_0,
+                arg: pattern6_1,
+                flags: pattern6_2,
+                offset: pattern6_3,
+            } = &pattern5_0
+            {
+                if let &Opcode::Load = &pattern6_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern6_2) {
+                        // Rule at src/isa/s390x/inst.isle line 1981.
+                        let expr0_0 = constructor_sink_load(ctx, pattern4_0)?;
+                        let expr1_0 =
+                            constructor_emit_sext64_mem(ctx, pattern0_0, pattern3_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 1983.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_sext64_reg(ctx, pattern0_0, pattern3_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) {
+        // Rule at src/isa/s390x/inst.isle line 1985.
+        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+        let expr1_0 = constructor_emit_mov(ctx, pattern3_0, pattern0_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term put_in_reg_zext32.
+pub fn constructor_put_in_reg_zext32<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::value_type(ctx, pattern0_0);
+    if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 1990.
+        let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?;
+        let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::fits_in_16(ctx, pattern1_0) {
+        if let Some(pattern3_0) = C::sinkable_inst(ctx, pattern0_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern3_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+                flags: pattern5_2,
+                offset: pattern5_3,
+            } = &pattern4_0
+            {
+                if let &Opcode::Load = &pattern5_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                        // Rule at src/isa/s390x/inst.isle line 1992.
+                        let expr0_0 = constructor_sink_load(ctx, pattern3_0)?;
+                        let expr1_0 = constructor_zext32_mem(ctx, pattern2_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 1994.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        let expr1_0 = constructor_zext32_reg(ctx, pattern2_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 1996.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term put_in_reg_sext32.
+pub fn constructor_put_in_reg_sext32<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::value_type(ctx, pattern0_0);
+    if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2001.
+        let expr0_0 = constructor_ty_ext32(ctx, pattern1_0)?;
+        let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::fits_in_16(ctx, pattern1_0) {
+        if let Some(pattern3_0) = C::sinkable_inst(ctx, pattern0_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern3_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+                flags: pattern5_2,
+                offset: pattern5_3,
+            } = &pattern4_0
+            {
+                if let &Opcode::Load = &pattern5_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                        // Rule at src/isa/s390x/inst.isle line 2003.
+                        let expr0_0 = constructor_sink_load(ctx, pattern3_0)?;
+                        let expr1_0 = constructor_sext32_mem(ctx, pattern2_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 2005.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        let expr1_0 = constructor_sext32_reg(ctx, pattern2_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::ty_32_or_64(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 2007.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term put_in_reg_zext64.
+pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::value_type(ctx, pattern0_0);
+    if let Some(pattern2_0) = C::u64_from_value(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2012.
+        let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?;
+        let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::gpr32_ty(ctx, pattern1_0) {
+        if let Some(pattern3_0) = C::sinkable_inst(ctx, pattern0_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern3_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+                flags: pattern5_2,
+                offset: pattern5_3,
+            } = &pattern4_0
+            {
+                if let &Opcode::Load = &pattern5_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                        // Rule at src/isa/s390x/inst.isle line 2014.
+                        let expr0_0 = constructor_sink_load(ctx, pattern3_0)?;
+                        let expr1_0 = constructor_zext64_mem(ctx, pattern2_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 2016.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        let expr1_0 = constructor_zext64_reg(ctx, pattern2_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 2018.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term put_in_reg_sext64.
+pub fn constructor_put_in_reg_sext64<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::value_type(ctx, pattern0_0);
+    if let Some(pattern2_0) = C::u64_from_signed_value(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2023.
+        let expr0_0 = constructor_ty_ext64(ctx, pattern1_0)?;
+        let expr1_0 = constructor_imm(ctx, expr0_0, pattern2_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::gpr32_ty(ctx, pattern1_0) {
+        if let Some(pattern3_0) = C::sinkable_inst(ctx, pattern0_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern3_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+                flags: pattern5_2,
+                offset: pattern5_3,
+            } = &pattern4_0
+            {
+                if let &Opcode::Load = &pattern5_0 {
+                    if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                        // Rule at src/isa/s390x/inst.isle line 2025.
+                        let expr0_0 = constructor_sink_load(ctx, pattern3_0)?;
+                        let expr1_0 = constructor_sext64_mem(ctx, pattern2_0, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/inst.isle line 2027.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        let expr1_0 = constructor_sext64_reg(ctx, pattern2_0, expr0_0)?;
+        return Some(expr1_0);
+    }
+    if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/inst.isle line 2029.
+        let expr0_0 = C::put_in_reg(ctx, pattern0_0);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term put_in_regpair_lo_zext32.
+pub fn constructor_put_in_regpair_lo_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: Value,
+    arg1: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2035.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?;
+    let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_put_in_reg_zext32(ctx, expr1_0, pattern0_0)?;
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term put_in_regpair_lo_sext32.
+pub fn constructor_put_in_regpair_lo_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Value,
+    arg1: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2043.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?;
+    let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_put_in_reg_sext32(ctx, expr1_0, pattern0_0)?;
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term put_in_regpair_lo_zext64.
+pub fn constructor_put_in_regpair_lo_zext64<C: Context>(
+    ctx: &mut C,
+    arg0: Value,
+    arg1: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2051.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?;
+    let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_put_in_reg_zext64(ctx, expr1_0, pattern0_0)?;
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term put_in_regpair_lo_sext64.
+pub fn constructor_put_in_regpair_lo_sext64<C: Context>(
+    ctx: &mut C,
+    arg0: Value,
+    arg1: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2059.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern1_0)?;
+    let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_put_in_reg_sext64(ctx, expr1_0, pattern0_0)?;
+    let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term emit_cmov_imm.
+pub fn constructor_emit_cmov_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: WritableReg,
+    arg2: &Cond,
+    arg3: i16,
+) -> Option<ConsumesFlags> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2069.
+        let expr0_0 = MInst::CMov32SImm16 {
+            rd: pattern2_0,
+            cond: pattern3_0.clone(),
+            imm: pattern4_0,
+        };
+        let expr1_0 = C::writable_reg_to_reg(ctx, pattern2_0);
+        let expr2_0 = ConsumesFlags::ConsumesFlags {
+            inst: expr0_0,
+            result: expr1_0,
+        };
+        return Some(expr2_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2072.
+        let expr0_0 = MInst::CMov64SImm16 {
+            rd: pattern2_0,
+            cond: pattern3_0.clone(),
+            imm: pattern4_0,
+        };
+        let expr1_0 = C::writable_reg_to_reg(ctx, pattern2_0);
+        let expr2_0 = ConsumesFlags::ConsumesFlags {
+            inst: expr0_0,
+            result: expr1_0,
+        };
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term cmov_imm.
+pub fn constructor_cmov_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &Cond,
+    arg2: i16,
+    arg3: Reg,
+) -> Option<ConsumesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 2078.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?;
+    let expr1_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term cmov_imm_regpair_lo.
+pub fn constructor_cmov_imm_regpair_lo<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ProducesFlags,
+    arg2: &Cond,
+    arg3: i16,
+    arg4: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 2085.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?;
+    let expr1_0 = constructor_writable_regpair_lo(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?;
+    let expr3_0 = constructor_with_flags_1(ctx, pattern1_0, &expr2_0)?;
+    let expr4_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term cmov_imm_regpair_hi.
+pub fn constructor_cmov_imm_regpair_hi<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ProducesFlags,
+    arg2: &Cond,
+    arg3: i16,
+    arg4: &RegPair,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 2094.
+    let expr0_0 = constructor_copy_writable_regpair(ctx, pattern4_0)?;
+    let expr1_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?;
+    let expr2_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr1_0, pattern2_0, pattern3_0)?;
+    let expr3_0 = constructor_with_flags_1(ctx, pattern1_0, &expr2_0)?;
+    let expr4_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term emit_cmov_reg.
+pub fn constructor_emit_cmov_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: WritableReg,
+    arg2: &Cond,
+    arg3: Reg,
+) -> Option<ConsumesFlags> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2108.
+        let expr0_0 = MInst::FpuCMov32 {
+            rd: pattern2_0,
+            cond: pattern3_0.clone(),
+            rm: pattern4_0,
+        };
+        let expr1_0 = C::writable_reg_to_reg(ctx, pattern2_0);
+        let expr2_0 = ConsumesFlags::ConsumesFlags {
+            inst: expr0_0,
+            result: expr1_0,
+        };
+        return Some(expr2_0);
+    }
+    if pattern0_0 == F64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2111.
+        let expr0_0 = MInst::FpuCMov64 {
+            rd: pattern2_0,
+            cond: pattern3_0.clone(),
+            rm: pattern4_0,
+        };
+        let expr1_0 = C::writable_reg_to_reg(ctx, pattern2_0);
+        let expr2_0 = ConsumesFlags::ConsumesFlags {
+            inst: expr0_0,
+            result: expr1_0,
+        };
+        return Some(expr2_0);
+    }
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2102.
+        let expr0_0 = MInst::CMov32 {
+            rd: pattern2_0,
+            cond: pattern3_0.clone(),
+            rm: pattern4_0,
+        };
+        let expr1_0 = C::writable_reg_to_reg(ctx, pattern2_0);
+        let expr2_0 = ConsumesFlags::ConsumesFlags {
+            inst: expr0_0,
+            result: expr1_0,
+        };
+        return Some(expr2_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2105.
+        let expr0_0 = MInst::CMov64 {
+            rd: pattern2_0,
+            cond: pattern3_0.clone(),
+            rm: pattern4_0,
+        };
+        let expr1_0 = C::writable_reg_to_reg(ctx, pattern2_0);
+        let expr2_0 = ConsumesFlags::ConsumesFlags {
+            inst: expr0_0,
+            result: expr1_0,
+        };
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term cmov_reg.
+pub fn constructor_cmov_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &Cond,
+    arg2: Reg,
+    arg3: Reg,
+) -> Option<ConsumesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 2117.
+    let expr0_0 = constructor_copy_writable_reg(ctx, pattern0_0, pattern3_0)?;
+    let expr1_0 = constructor_emit_cmov_reg(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term trap_if.
+pub fn constructor_trap_if<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &Cond,
+    arg2: &TrapCode,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2125.
+        let expr0_0 = C::emit(ctx, &pattern1_0);
+        let expr1_0 = MInst::TrapIf {
+            cond: pattern2_0.clone(),
+            trap_code: pattern3_0.clone(),
+        };
+        let expr2_0 = C::emit(ctx, &expr1_0);
+        return Some(pattern1_1);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term icmps_reg_and_trap.
+pub fn constructor_icmps_reg_and_trap<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+    arg3: &Cond,
+    arg4: &TrapCode,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 2131.
+    let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?;
+    let expr1_0 = MInst::CmpTrapRR {
+        op: expr0_0,
+        rn: pattern1_0,
+        rm: pattern2_0,
+        cond: pattern3_0.clone(),
+        trap_code: pattern4_0.clone(),
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::invalid_reg(ctx);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term icmps_simm16_and_trap.
+pub fn constructor_icmps_simm16_and_trap<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: i16,
+    arg3: &Cond,
+    arg4: &TrapCode,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 2137.
+    let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?;
+    let expr1_0 = MInst::CmpTrapRSImm16 {
+        op: expr0_0,
+        rn: pattern1_0,
+        imm: pattern2_0,
+        cond: pattern3_0.clone(),
+        trap_code: pattern4_0.clone(),
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::invalid_reg(ctx);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term icmpu_reg_and_trap.
+pub fn constructor_icmpu_reg_and_trap<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+    arg3: &Cond,
+    arg4: &TrapCode,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 2143.
+    let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?;
+    let expr1_0 = MInst::CmpTrapRR {
+        op: expr0_0,
+        rn: pattern1_0,
+        rm: pattern2_0,
+        cond: pattern3_0.clone(),
+        trap_code: pattern4_0.clone(),
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::invalid_reg(ctx);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term icmpu_uimm16_and_trap.
+pub fn constructor_icmpu_uimm16_and_trap<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u16,
+    arg3: &Cond,
+    arg4: &TrapCode,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    let pattern4_0 = arg4;
+    // Rule at src/isa/s390x/inst.isle line 2149.
+    let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?;
+    let expr1_0 = MInst::CmpTrapRUImm16 {
+        op: expr0_0,
+        rn: pattern1_0,
+        imm: pattern2_0,
+        cond: pattern3_0.clone(),
+        trap_code: pattern4_0.clone(),
+    };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = C::invalid_reg(ctx);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term bool.
+pub fn constructor_bool<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesFlags,
+    arg1: &Cond,
+) -> Option<ProducesBool> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2162.
+    let expr0_0 = ProducesBool::ProducesBool {
+        producer: pattern0_0.clone(),
+        cond: pattern1_0.clone(),
+    };
+    return Some(expr0_0);
+}
+
+// Generated as internal constructor for term invert_bool.
+pub fn constructor_invert_bool<C: Context>(
+    ctx: &mut C,
+    arg0: &ProducesBool,
+) -> Option<ProducesBool> {
+    let pattern0_0 = arg0;
+    if let &ProducesBool::ProducesBool {
+        producer: ref pattern1_0,
+        cond: ref pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 2166.
+        let expr0_0 = C::invert_cond(ctx, &pattern1_1);
+        let expr1_0 = constructor_bool(ctx, &pattern1_0, &expr0_0)?;
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_producer.
+pub fn constructor_emit_producer<C: Context>(ctx: &mut C, arg0: &ProducesFlags) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    if let &ProducesFlags::ProducesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 2175.
+        let expr0_0 = C::emit(ctx, &pattern1_0);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term emit_consumer.
+pub fn constructor_emit_consumer<C: Context>(ctx: &mut C, arg0: &ConsumesFlags) -> Option<Unit> {
+    let pattern0_0 = arg0;
+    if let &ConsumesFlags::ConsumesFlags {
+        inst: ref pattern1_0,
+        result: pattern1_1,
+    } = pattern0_0
+    {
+        // Rule at src/isa/s390x/inst.isle line 2177.
+        let expr0_0 = C::emit(ctx, &pattern1_0);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term select_bool_reg.
+pub fn constructor_select_bool_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ProducesBool,
+    arg2: Reg,
+    arg3: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if let &ProducesBool::ProducesBool {
+        producer: ref pattern2_0,
+        cond: ref pattern2_1,
+    } = pattern1_0
+    {
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2181.
+        let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+        let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?;
+        let expr2_0 = constructor_emit_mov(ctx, pattern0_0, expr0_0, pattern4_0)?;
+        let expr3_0 = constructor_emit_cmov_reg(ctx, pattern0_0, expr0_0, &pattern2_1, pattern3_0)?;
+        let expr4_0 = constructor_emit_consumer(ctx, &expr3_0)?;
+        let expr5_0 = C::writable_reg_to_reg(ctx, expr0_0);
+        return Some(expr5_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term select_bool_imm.
+pub fn constructor_select_bool_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ProducesBool,
+    arg2: i16,
+    arg3: u64,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if let &ProducesBool::ProducesBool {
+        producer: ref pattern2_0,
+        cond: ref pattern2_1,
+    } = pattern1_0
+    {
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/inst.isle line 2190.
+        let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
+        let expr1_0 = constructor_emit_producer(ctx, &pattern2_0)?;
+        let expr2_0 = constructor_emit_imm(ctx, pattern0_0, expr0_0, pattern4_0)?;
+        let expr3_0 = constructor_emit_cmov_imm(ctx, pattern0_0, expr0_0, &pattern2_1, pattern3_0)?;
+        let expr4_0 = constructor_emit_consumer(ctx, &expr3_0)?;
+        let expr5_0 = C::writable_reg_to_reg(ctx, expr0_0);
+        return Some(expr5_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term lower_bool.
+pub fn constructor_lower_bool<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &ProducesBool,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == B1 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2200.
+        let expr0_0: Type = B1;
+        let expr1_0: i16 = 1;
+        let expr2_0: u64 = 0;
+        let expr3_0 = constructor_select_bool_imm(ctx, expr0_0, pattern2_0, expr1_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    if pattern0_0 == B8 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2201.
+        let expr0_0: Type = B8;
+        let expr1_0: i16 = -1;
+        let expr2_0: u64 = 0;
+        let expr3_0 = constructor_select_bool_imm(ctx, expr0_0, pattern2_0, expr1_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    if pattern0_0 == B16 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2202.
+        let expr0_0: Type = B16;
+        let expr1_0: i16 = -1;
+        let expr2_0: u64 = 0;
+        let expr3_0 = constructor_select_bool_imm(ctx, expr0_0, pattern2_0, expr1_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    if pattern0_0 == B32 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2203.
+        let expr0_0: Type = B32;
+        let expr1_0: i16 = -1;
+        let expr2_0: u64 = 0;
+        let expr3_0 = constructor_select_bool_imm(ctx, expr0_0, pattern2_0, expr1_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    if pattern0_0 == B64 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2204.
+        let expr0_0: Type = B64;
+        let expr1_0: i16 = -1;
+        let expr2_0: u64 = 0;
+        let expr3_0 = constructor_select_bool_imm(ctx, expr0_0, pattern2_0, expr1_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term clz_reg.
+pub fn constructor_clz_reg<C: Context>(ctx: &mut C, arg0: i16, arg1: Reg) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == 64 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2213.
+        let expr0_0 = constructor_temp_writable_regpair(ctx)?;
+        let expr1_0 = MInst::Flogr { rn: pattern2_0 };
+        let expr2_0 = C::emit(ctx, &expr1_0);
+        let expr3_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+        return Some(expr3_0);
+    }
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2222.
+    let expr0_0 = constructor_temp_writable_regpair(ctx)?;
+    let expr1_0 = MInst::Flogr { rn: pattern1_0 };
+    let expr2_0 = C::emit(ctx, &expr1_0);
+    let expr3_0 = constructor_writable_regpair_hi(ctx, &expr0_0)?;
+    let expr4_0 = IntCC::Equal;
+    let expr5_0 = C::intcc_as_cond(ctx, &expr4_0);
+    let expr6_0 = MInst::CMov64SImm16 {
+        rd: expr3_0,
+        cond: expr5_0,
+        imm: pattern0_0,
+    };
+    let expr7_0 = C::emit(ctx, &expr6_0);
+    let expr8_0 = constructor_writable_regpair_to_regpair(ctx, &expr0_0)?;
+    return Some(expr8_0);
+}
+
+// Generated as internal constructor for term aluop_add.
+pub fn constructor_aluop_add<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/inst.isle line 2233.
+        let expr0_0 = ALUOp::Add32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2234.
+        let expr0_0 = ALUOp::Add32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2235.
+        let expr0_0 = ALUOp::Add32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2236.
+        let expr0_0 = ALUOp::Add64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_add_sext16.
+pub fn constructor_aluop_add_sext16<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2239.
+        let expr0_0 = ALUOp::Add32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2240.
+        let expr0_0 = ALUOp::Add32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2241.
+        let expr0_0 = ALUOp::Add64Ext16;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_add_sext32.
+pub fn constructor_aluop_add_sext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2244.
+        let expr0_0 = ALUOp::Add64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term add_reg.
+pub fn constructor_add_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2247.
+    let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_reg_sext32.
+pub fn constructor_add_reg_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2250.
+    let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_simm16.
+pub fn constructor_add_simm16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: i16,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2253.
+    let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_simm32.
+pub fn constructor_add_simm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: i32,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2256.
+    let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_mem.
+pub fn constructor_add_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2259.
+    let expr0_0 = constructor_aluop_add(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_mem_sext16.
+pub fn constructor_add_mem_sext16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2262.
+    let expr0_0 = constructor_aluop_add_sext16(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_mem_sext32.
+pub fn constructor_add_mem_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2265.
+    let expr0_0 = constructor_aluop_add_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_add_logical.
+pub fn constructor_aluop_add_logical<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2271.
+        let expr0_0 = ALUOp::AddLogical32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2272.
+        let expr0_0 = ALUOp::AddLogical64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_add_logical_zext32.
+pub fn constructor_aluop_add_logical_zext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2275.
+        let expr0_0 = ALUOp::AddLogical64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term add_logical_reg.
+pub fn constructor_add_logical_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2278.
+    let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_logical_reg_zext32.
+pub fn constructor_add_logical_reg_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2281.
+    let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_logical_zimm32.
+pub fn constructor_add_logical_zimm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u32,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2284.
+    let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_logical_mem.
+pub fn constructor_add_logical_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2287.
+    let expr0_0 = constructor_aluop_add_logical(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term add_logical_mem_zext32.
+pub fn constructor_add_logical_mem_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2290.
+    let expr0_0 = constructor_aluop_add_logical_zext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_sub.
+pub fn constructor_aluop_sub<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/inst.isle line 2296.
+        let expr0_0 = ALUOp::Sub32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2297.
+        let expr0_0 = ALUOp::Sub32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2298.
+        let expr0_0 = ALUOp::Sub32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2299.
+        let expr0_0 = ALUOp::Sub64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_sub_sext16.
+pub fn constructor_aluop_sub_sext16<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2302.
+        let expr0_0 = ALUOp::Sub32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2303.
+        let expr0_0 = ALUOp::Sub32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2304.
+        let expr0_0 = ALUOp::Sub64Ext16;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_sub_sext32.
+pub fn constructor_aluop_sub_sext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2307.
+        let expr0_0 = ALUOp::Sub64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sub_reg.
+pub fn constructor_sub_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2310.
+    let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_reg_sext32.
+pub fn constructor_sub_reg_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2313.
+    let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_mem.
+pub fn constructor_sub_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2316.
+    let expr0_0 = constructor_aluop_sub(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_mem_sext16.
+pub fn constructor_sub_mem_sext16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2319.
+    let expr0_0 = constructor_aluop_sub_sext16(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_mem_sext32.
+pub fn constructor_sub_mem_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2322.
+    let expr0_0 = constructor_aluop_sub_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_sub_logical.
+pub fn constructor_aluop_sub_logical<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2328.
+        let expr0_0 = ALUOp::SubLogical32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2329.
+        let expr0_0 = ALUOp::SubLogical64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_sub_logical_zext32.
+pub fn constructor_aluop_sub_logical_zext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2332.
+        let expr0_0 = ALUOp::SubLogical64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sub_logical_reg.
+pub fn constructor_sub_logical_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2335.
+    let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_logical_reg_zext32.
+pub fn constructor_sub_logical_reg_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2338.
+    let expr0_0 = constructor_aluop_sub_logical_zext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_logical_zimm32.
+pub fn constructor_sub_logical_zimm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u32,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2341.
+    let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_ruimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_logical_mem.
+pub fn constructor_sub_logical_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2344.
+    let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sub_logical_mem_zext32.
+pub fn constructor_sub_logical_mem_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2347.
+    let expr0_0 = constructor_aluop_sub_logical(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_mul.
+pub fn constructor_aluop_mul<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/inst.isle line 2353.
+        let expr0_0 = ALUOp::Mul32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2354.
+        let expr0_0 = ALUOp::Mul32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2355.
+        let expr0_0 = ALUOp::Mul32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2356.
+        let expr0_0 = ALUOp::Mul64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_mul_sext16.
+pub fn constructor_aluop_mul_sext16<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2359.
+        let expr0_0 = ALUOp::Mul32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2360.
+        let expr0_0 = ALUOp::Mul32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2361.
+        let expr0_0 = ALUOp::Mul64Ext16;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_mul_sext32.
+pub fn constructor_aluop_mul_sext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2364.
+        let expr0_0 = ALUOp::Mul64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term mul_reg.
+pub fn constructor_mul_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2367.
+    let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term mul_reg_sext32.
+pub fn constructor_mul_reg_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2370.
+    let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term mul_simm16.
+pub fn constructor_mul_simm16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: i16,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2373.
+    let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rsimm16(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term mul_simm32.
+pub fn constructor_mul_simm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: i32,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2376.
+    let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rsimm32(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term mul_mem.
+pub fn constructor_mul_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2379.
+    let expr0_0 = constructor_aluop_mul(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term mul_mem_sext16.
+pub fn constructor_mul_mem_sext16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2382.
+    let expr0_0 = constructor_aluop_mul_sext16(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term mul_mem_sext32.
+pub fn constructor_mul_mem_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2385.
+    let expr0_0 = constructor_aluop_mul_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term udivmod.
+pub fn constructor_udivmod<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &RegPair,
+    arg2: Reg,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2391.
+        let expr0_0 = constructor_udivmod32(ctx, pattern2_0, pattern3_0)?;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2392.
+        let expr0_0 = constructor_udivmod64(ctx, pattern2_0, pattern3_0)?;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sdivmod.
+pub fn constructor_sdivmod<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: &RegPair,
+    arg2: Reg,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2398.
+        let expr0_0 = constructor_sdivmod32(ctx, pattern2_0, pattern3_0)?;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2399.
+        let expr0_0 = constructor_sdivmod64(ctx, pattern2_0, pattern3_0)?;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_and.
+pub fn constructor_aluop_and<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2405.
+        let expr0_0 = ALUOp::And32;
+        return Some(expr0_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2406.
+        let expr0_0 = ALUOp::And64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term and_reg.
+pub fn constructor_and_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2409.
+    let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term and_uimm16shifted.
+pub fn constructor_and_uimm16shifted<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: UImm16Shifted,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2412.
+    let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?;
+    let expr1_0 =
+        constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term and_uimm32shifted.
+pub fn constructor_and_uimm32shifted<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: UImm32Shifted,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2415.
+    let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?;
+    let expr1_0 =
+        constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term and_mem.
+pub fn constructor_and_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2418.
+    let expr0_0 = constructor_aluop_and(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_or.
+pub fn constructor_aluop_or<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2424.
+        let expr0_0 = ALUOp::Orr32;
+        return Some(expr0_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2425.
+        let expr0_0 = ALUOp::Orr64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term or_reg.
+pub fn constructor_or_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2428.
+    let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term or_uimm16shifted.
+pub fn constructor_or_uimm16shifted<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: UImm16Shifted,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2431.
+    let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?;
+    let expr1_0 =
+        constructor_alu_ruimm16shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term or_uimm32shifted.
+pub fn constructor_or_uimm32shifted<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: UImm32Shifted,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2434.
+    let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?;
+    let expr1_0 =
+        constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term or_mem.
+pub fn constructor_or_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2437.
+    let expr0_0 = constructor_aluop_or(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_xor.
+pub fn constructor_aluop_xor<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2443.
+        let expr0_0 = ALUOp::Xor32;
+        return Some(expr0_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2444.
+        let expr0_0 = ALUOp::Xor64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term xor_reg.
+pub fn constructor_xor_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2447.
+    let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term xor_uimm32shifted.
+pub fn constructor_xor_uimm32shifted<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: UImm32Shifted,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2450.
+    let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?;
+    let expr1_0 =
+        constructor_alu_ruimm32shifted(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term xor_mem.
+pub fn constructor_xor_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2453.
+    let expr0_0 = constructor_aluop_xor(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rx(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term not_reg.
+pub fn constructor_not_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2459.
+        let expr0_0: u32 = 4294967295;
+        let expr1_0: u8 = 0;
+        let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0);
+        let expr3_0 = constructor_xor_uimm32shifted(ctx, pattern1_0, pattern2_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/inst.isle line 2461.
+        let expr0_0: u32 = 4294967295;
+        let expr1_0: u8 = 0;
+        let expr2_0 = C::uimm32shifted(ctx, expr0_0, expr1_0);
+        let expr3_0 = constructor_xor_uimm32shifted(ctx, pattern1_0, pattern2_0, expr2_0)?;
+        let expr4_0: u32 = 4294967295;
+        let expr5_0: u8 = 32;
+        let expr6_0 = C::uimm32shifted(ctx, expr4_0, expr5_0);
+        let expr7_0 = constructor_xor_uimm32shifted(ctx, pattern1_0, expr3_0, expr6_0)?;
+        return Some(expr7_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term aluop_and_not.
+pub fn constructor_aluop_and_not<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2470.
+        let expr0_0 = ALUOp::AndNot32;
+        return Some(expr0_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2471.
+        let expr0_0 = ALUOp::AndNot64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term and_not_reg.
+pub fn constructor_and_not_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2474.
+    let expr0_0 = constructor_aluop_and_not(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_or_not.
+pub fn constructor_aluop_or_not<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2480.
+        let expr0_0 = ALUOp::OrrNot32;
+        return Some(expr0_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2481.
+        let expr0_0 = ALUOp::OrrNot64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term or_not_reg.
+pub fn constructor_or_not_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2484.
+    let expr0_0 = constructor_aluop_or_not(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term aluop_xor_not.
+pub fn constructor_aluop_xor_not<C: Context>(ctx: &mut C, arg0: Type) -> Option<ALUOp> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2490.
+        let expr0_0 = ALUOp::XorNot32;
+        return Some(expr0_0);
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        // Rule at src/isa/s390x/inst.isle line 2491.
+        let expr0_0 = ALUOp::XorNot64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term xor_not_reg.
+pub fn constructor_xor_not_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2494.
+    let expr0_0 = constructor_aluop_xor_not(ctx, pattern0_0)?;
+    let expr1_0 = constructor_alu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term unaryop_abs.
+pub fn constructor_unaryop_abs<C: Context>(ctx: &mut C, arg0: Type) -> Option<UnaryOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2500.
+        let expr0_0 = UnaryOp::Abs32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2501.
+        let expr0_0 = UnaryOp::Abs64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term unaryop_abs_sext32.
+pub fn constructor_unaryop_abs_sext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<UnaryOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2504.
+        let expr0_0 = UnaryOp::Abs64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term abs_reg.
+pub fn constructor_abs_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2507.
+    let expr0_0 = constructor_unaryop_abs(ctx, pattern0_0)?;
+    let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term abs_reg_sext32.
+pub fn constructor_abs_reg_sext32<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2510.
+    let expr0_0 = constructor_unaryop_abs_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term unaryop_neg.
+pub fn constructor_unaryop_neg<C: Context>(ctx: &mut C, arg0: Type) -> Option<UnaryOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/inst.isle line 2516.
+        let expr0_0 = UnaryOp::Neg32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2517.
+        let expr0_0 = UnaryOp::Neg32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2518.
+        let expr0_0 = UnaryOp::Neg32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2519.
+        let expr0_0 = UnaryOp::Neg64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term unaryop_neg_sext32.
+pub fn constructor_unaryop_neg_sext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<UnaryOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2522.
+        let expr0_0 = UnaryOp::Neg64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term neg_reg.
+pub fn constructor_neg_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2525.
+    let expr0_0 = constructor_unaryop_neg(ctx, pattern0_0)?;
+    let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term neg_reg_sext32.
+pub fn constructor_neg_reg_sext32<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2528.
+    let expr0_0 = constructor_unaryop_neg_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_unary_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term shiftop_rot.
+pub fn constructor_shiftop_rot<C: Context>(ctx: &mut C, arg0: Type) -> Option<ShiftOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2534.
+        let expr0_0 = ShiftOp::RotL32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2535.
+        let expr0_0 = ShiftOp::RotL64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term rot_reg.
+pub fn constructor_rot_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2538.
+    let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?;
+    let expr1_0: u8 = 0;
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term rot_imm.
+pub fn constructor_rot_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u8,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2542.
+    let expr0_0 = constructor_shiftop_rot(ctx, pattern0_0)?;
+    let expr1_0 = C::zero_reg(ctx);
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term shiftop_lshl.
+pub fn constructor_shiftop_lshl<C: Context>(ctx: &mut C, arg0: Type) -> Option<ShiftOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/inst.isle line 2549.
+        let expr0_0 = ShiftOp::LShL32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/inst.isle line 2550.
+        let expr0_0 = ShiftOp::LShL32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2551.
+        let expr0_0 = ShiftOp::LShL32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2552.
+        let expr0_0 = ShiftOp::LShL64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term lshl_reg.
+pub fn constructor_lshl_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2555.
+    let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?;
+    let expr1_0: u8 = 0;
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term lshl_imm.
+pub fn constructor_lshl_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u8,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2559.
+    let expr0_0 = constructor_shiftop_lshl(ctx, pattern0_0)?;
+    let expr1_0 = C::zero_reg(ctx);
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term shiftop_lshr.
+pub fn constructor_shiftop_lshr<C: Context>(ctx: &mut C, arg0: Type) -> Option<ShiftOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2566.
+        let expr0_0 = ShiftOp::LShR32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2567.
+        let expr0_0 = ShiftOp::LShR64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term lshr_reg.
+pub fn constructor_lshr_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2570.
+    let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?;
+    let expr1_0: u8 = 0;
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term lshr_imm.
+pub fn constructor_lshr_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u8,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2574.
+    let expr0_0 = constructor_shiftop_lshr(ctx, pattern0_0)?;
+    let expr1_0 = C::zero_reg(ctx);
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term shiftop_ashr.
+pub fn constructor_shiftop_ashr<C: Context>(ctx: &mut C, arg0: Type) -> Option<ShiftOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2581.
+        let expr0_0 = ShiftOp::AShR32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2582.
+        let expr0_0 = ShiftOp::AShR64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term ashr_reg.
+pub fn constructor_ashr_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2585.
+    let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?;
+    let expr1_0: u8 = 0;
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, expr1_0, pattern2_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term ashr_imm.
+pub fn constructor_ashr_imm<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u8,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2589.
+    let expr0_0 = constructor_shiftop_ashr(ctx, pattern0_0)?;
+    let expr1_0 = C::zero_reg(ctx);
+    let expr2_0 = constructor_shift_rr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0, expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term popcnt_byte.
+pub fn constructor_popcnt_byte<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 2596.
+    let expr0_0: Type = I64;
+    let expr1_0 = UnaryOp::PopcntByte;
+    let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term popcnt_reg.
+pub fn constructor_popcnt_reg<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/inst.isle line 2599.
+    let expr0_0: Type = I64;
+    let expr1_0 = UnaryOp::PopcntReg;
+    let expr2_0 = constructor_unary_rr(ctx, expr0_0, &expr1_0, pattern0_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term atomic_rmw_and.
+pub fn constructor_atomic_rmw_and<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2605.
+        let expr0_0: Type = I32;
+        let expr1_0 = ALUOp::And32;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2606.
+        let expr0_0: Type = I64;
+        let expr1_0 = ALUOp::And64;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term atomic_rmw_or.
+pub fn constructor_atomic_rmw_or<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2609.
+        let expr0_0: Type = I32;
+        let expr1_0 = ALUOp::Orr32;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2610.
+        let expr0_0: Type = I64;
+        let expr1_0 = ALUOp::Orr64;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term atomic_rmw_xor.
+pub fn constructor_atomic_rmw_xor<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2613.
+        let expr0_0: Type = I32;
+        let expr1_0 = ALUOp::Xor32;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2614.
+        let expr0_0: Type = I64;
+        let expr1_0 = ALUOp::Xor64;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term atomic_rmw_add.
+pub fn constructor_atomic_rmw_add<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2617.
+        let expr0_0: Type = I32;
+        let expr1_0 = ALUOp::Add32;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2618.
+        let expr0_0: Type = I64;
+        let expr1_0 = ALUOp::Add64;
+        let expr2_0 = constructor_atomic_rmw_impl(ctx, expr0_0, &expr1_0, pattern2_0, pattern3_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fpuop2_add.
+pub fn constructor_fpuop2_add<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp2> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2624.
+        let expr0_0 = FPUOp2::Add32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2625.
+        let expr0_0 = FPUOp2::Add64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fadd_reg.
+pub fn constructor_fadd_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2628.
+    let expr0_0 = constructor_fpuop2_add(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop2_sub.
+pub fn constructor_fpuop2_sub<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp2> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2634.
+        let expr0_0 = FPUOp2::Sub32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2635.
+        let expr0_0 = FPUOp2::Sub64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fsub_reg.
+pub fn constructor_fsub_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2638.
+    let expr0_0 = constructor_fpuop2_sub(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop2_mul.
+pub fn constructor_fpuop2_mul<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp2> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2644.
+        let expr0_0 = FPUOp2::Mul32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2645.
+        let expr0_0 = FPUOp2::Mul64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fmul_reg.
+pub fn constructor_fmul_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2648.
+    let expr0_0 = constructor_fpuop2_mul(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop2_div.
+pub fn constructor_fpuop2_div<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp2> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2654.
+        let expr0_0 = FPUOp2::Div32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2655.
+        let expr0_0 = FPUOp2::Div64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fdiv_reg.
+pub fn constructor_fdiv_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2658.
+    let expr0_0 = constructor_fpuop2_div(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop2_min.
+pub fn constructor_fpuop2_min<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp2> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2664.
+        let expr0_0 = FPUOp2::Min32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2665.
+        let expr0_0 = FPUOp2::Min64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fmin_reg.
+pub fn constructor_fmin_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2668.
+    let expr0_0 = constructor_fpuop2_min(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop2_max.
+pub fn constructor_fpuop2_max<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp2> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2674.
+        let expr0_0 = FPUOp2::Max32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2675.
+        let expr0_0 = FPUOp2::Max64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fmax_reg.
+pub fn constructor_fmax_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2678.
+    let expr0_0 = constructor_fpuop2_max(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpuvec_rrr(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop3_fma.
+pub fn constructor_fpuop3_fma<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp3> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2684.
+        let expr0_0 = FPUOp3::MAdd32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2685.
+        let expr0_0 = FPUOp3::MAdd64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fma_reg.
+pub fn constructor_fma_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+    arg3: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/inst.isle line 2688.
+    let expr0_0 = constructor_fpuop3_fma(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rrrr(
+        ctx, pattern0_0, &expr0_0, pattern3_0, pattern1_0, pattern2_0,
+    )?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop1_sqrt.
+pub fn constructor_fpuop1_sqrt<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp1> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2694.
+        let expr0_0 = FPUOp1::Sqrt32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2695.
+        let expr0_0 = FPUOp1::Sqrt64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term sqrt_reg.
+pub fn constructor_sqrt_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2698.
+    let expr0_0 = constructor_fpuop1_sqrt(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop1_neg.
+pub fn constructor_fpuop1_neg<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp1> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2704.
+        let expr0_0 = FPUOp1::Neg32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2705.
+        let expr0_0 = FPUOp1::Neg64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fneg_reg.
+pub fn constructor_fneg_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2708.
+    let expr0_0 = constructor_fpuop1_neg(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop1_abs.
+pub fn constructor_fpuop1_abs<C: Context>(ctx: &mut C, arg0: Type) -> Option<FPUOp1> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2714.
+        let expr0_0 = FPUOp1::Abs32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2715.
+        let expr0_0 = FPUOp1::Abs64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fabs_reg.
+pub fn constructor_fabs_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2718.
+    let expr0_0 = constructor_fpuop1_abs(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuroundmode_ceil.
+pub fn constructor_fpuroundmode_ceil<C: Context>(ctx: &mut C, arg0: Type) -> Option<FpuRoundMode> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2724.
+        let expr0_0 = FpuRoundMode::Plus32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2725.
+        let expr0_0 = FpuRoundMode::Plus64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term ceil_reg.
+pub fn constructor_ceil_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2728.
+    let expr0_0 = constructor_fpuroundmode_ceil(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuroundmode_floor.
+pub fn constructor_fpuroundmode_floor<C: Context>(ctx: &mut C, arg0: Type) -> Option<FpuRoundMode> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2734.
+        let expr0_0 = FpuRoundMode::Minus32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2735.
+        let expr0_0 = FpuRoundMode::Minus64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term floor_reg.
+pub fn constructor_floor_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2738.
+    let expr0_0 = constructor_fpuroundmode_floor(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuroundmode_trunc.
+pub fn constructor_fpuroundmode_trunc<C: Context>(ctx: &mut C, arg0: Type) -> Option<FpuRoundMode> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2744.
+        let expr0_0 = FpuRoundMode::Zero32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2745.
+        let expr0_0 = FpuRoundMode::Zero64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term trunc_reg.
+pub fn constructor_trunc_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2748.
+    let expr0_0 = constructor_fpuroundmode_trunc(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuroundmode_nearest.
+pub fn constructor_fpuroundmode_nearest<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+) -> Option<FpuRoundMode> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        // Rule at src/isa/s390x/inst.isle line 2754.
+        let expr0_0 = FpuRoundMode::Nearest32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        // Rule at src/isa/s390x/inst.isle line 2755.
+        let expr0_0 = FpuRoundMode::Nearest64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term nearest_reg.
+pub fn constructor_nearest_reg<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    // Rule at src/isa/s390x/inst.isle line 2758.
+    let expr0_0 = constructor_fpuroundmode_nearest(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fpu_round(ctx, pattern0_0, &expr0_0, pattern1_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop1_promote.
+pub fn constructor_fpuop1_promote<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+) -> Option<FPUOp1> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F64 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == F32 {
+            // Rule at src/isa/s390x/inst.isle line 2764.
+            let expr0_0 = FPUOp1::Cvt32To64;
+            return Some(expr0_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fpromote_reg.
+pub fn constructor_fpromote_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2767.
+    let expr0_0 = constructor_fpuop1_promote(ctx, pattern0_0, pattern1_0)?;
+    let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpuop1_demote.
+pub fn constructor_fpuop1_demote<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+) -> Option<FPUOp1> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == F64 {
+            // Rule at src/isa/s390x/inst.isle line 2774.
+            let expr0_0 = FPUOp1::Cvt64To32;
+            return Some(expr0_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fdemote_reg.
+pub fn constructor_fdemote_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2777.
+    let expr0_0 = constructor_fpuop1_demote(ctx, pattern0_0, pattern1_0)?;
+    let expr1_0 = constructor_fpu_rr(ctx, pattern0_0, &expr0_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term uint_to_fpu_op.
+pub fn constructor_uint_to_fpu_op<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+) -> Option<IntToFpuOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == I32 {
+            // Rule at src/isa/s390x/inst.isle line 2784.
+            let expr0_0 = IntToFpuOp::U32ToF32;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == I64 {
+            // Rule at src/isa/s390x/inst.isle line 2786.
+            let expr0_0 = IntToFpuOp::U64ToF32;
+            return Some(expr0_0);
+        }
+    }
+    if pattern0_0 == F64 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == I32 {
+            // Rule at src/isa/s390x/inst.isle line 2785.
+            let expr0_0 = IntToFpuOp::U32ToF64;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == I64 {
+            // Rule at src/isa/s390x/inst.isle line 2787.
+            let expr0_0 = IntToFpuOp::U64ToF64;
+            return Some(expr0_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fcvt_from_uint_reg.
+pub fn constructor_fcvt_from_uint_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2790.
+    let expr0_0 = constructor_uint_to_fpu_op(ctx, pattern0_0, pattern1_0)?;
+    let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term sint_to_fpu_op.
+pub fn constructor_sint_to_fpu_op<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+) -> Option<IntToFpuOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == I32 {
+            // Rule at src/isa/s390x/inst.isle line 2797.
+            let expr0_0 = IntToFpuOp::I32ToF32;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == I64 {
+            // Rule at src/isa/s390x/inst.isle line 2799.
+            let expr0_0 = IntToFpuOp::I64ToF32;
+            return Some(expr0_0);
+        }
+    }
+    if pattern0_0 == F64 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == I32 {
+            // Rule at src/isa/s390x/inst.isle line 2798.
+            let expr0_0 = IntToFpuOp::I32ToF64;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == I64 {
+            // Rule at src/isa/s390x/inst.isle line 2800.
+            let expr0_0 = IntToFpuOp::I64ToF64;
+            return Some(expr0_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fcvt_from_sint_reg.
+pub fn constructor_fcvt_from_sint_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2803.
+    let expr0_0 = constructor_sint_to_fpu_op(ctx, pattern0_0, pattern1_0)?;
+    let expr1_0 = constructor_int_to_fpu(ctx, pattern0_0, &expr0_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpu_to_uint_op.
+pub fn constructor_fpu_to_uint_op<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+) -> Option<FpuToIntOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == F32 {
+            // Rule at src/isa/s390x/inst.isle line 2810.
+            let expr0_0 = FpuToIntOp::F32ToU32;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == F64 {
+            // Rule at src/isa/s390x/inst.isle line 2811.
+            let expr0_0 = FpuToIntOp::F64ToU32;
+            return Some(expr0_0);
+        }
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == F32 {
+            // Rule at src/isa/s390x/inst.isle line 2812.
+            let expr0_0 = FpuToIntOp::F32ToU64;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == F64 {
+            // Rule at src/isa/s390x/inst.isle line 2813.
+            let expr0_0 = FpuToIntOp::F64ToU64;
+            return Some(expr0_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fcvt_to_uint_reg_with_flags.
+pub fn constructor_fcvt_to_uint_reg_with_flags<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2816.
+    let expr0_0 = constructor_fpu_to_uint_op(ctx, pattern0_0, pattern1_0)?;
+    let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fcvt_to_uint_reg.
+pub fn constructor_fcvt_to_uint_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2820.
+    let expr0_0 = constructor_fcvt_to_uint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?;
+    let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fpu_to_sint_op.
+pub fn constructor_fpu_to_sint_op<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+) -> Option<FpuToIntOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == F32 {
+            // Rule at src/isa/s390x/inst.isle line 2827.
+            let expr0_0 = FpuToIntOp::F32ToI32;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == F64 {
+            // Rule at src/isa/s390x/inst.isle line 2828.
+            let expr0_0 = FpuToIntOp::F64ToI32;
+            return Some(expr0_0);
+        }
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        if pattern2_0 == F32 {
+            // Rule at src/isa/s390x/inst.isle line 2829.
+            let expr0_0 = FpuToIntOp::F32ToI64;
+            return Some(expr0_0);
+        }
+        if pattern2_0 == F64 {
+            // Rule at src/isa/s390x/inst.isle line 2830.
+            let expr0_0 = FpuToIntOp::F64ToI64;
+            return Some(expr0_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fcvt_to_sint_reg_with_flags.
+pub fn constructor_fcvt_to_sint_reg_with_flags<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2833.
+    let expr0_0 = constructor_fpu_to_sint_op(ctx, pattern0_0, pattern1_0)?;
+    let expr1_0 = constructor_fpu_to_int(ctx, pattern0_0, &expr0_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fcvt_to_sint_reg.
+pub fn constructor_fcvt_to_sint_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2837.
+    let expr0_0 = constructor_fcvt_to_sint_reg_with_flags(ctx, pattern0_0, pattern1_0, pattern2_0)?;
+    let expr1_0 = constructor_drop_flags(ctx, &expr0_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term cmpop_cmps.
+pub fn constructor_cmpop_cmps<C: Context>(ctx: &mut C, arg0: Type) -> Option<CmpOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2844.
+        let expr0_0 = CmpOp::CmpS32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2845.
+        let expr0_0 = CmpOp::CmpS64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term cmpop_cmps_sext16.
+pub fn constructor_cmpop_cmps_sext16<C: Context>(ctx: &mut C, arg0: Type) -> Option<CmpOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2848.
+        let expr0_0 = CmpOp::CmpS32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2849.
+        let expr0_0 = CmpOp::CmpS64Ext16;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term cmpop_cmps_sext32.
+pub fn constructor_cmpop_cmps_sext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<CmpOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2852.
+        let expr0_0 = CmpOp::CmpS64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term icmps_reg.
+pub fn constructor_icmps_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2855.
+    let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmps_reg_sext32.
+pub fn constructor_icmps_reg_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2858.
+    let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmps_simm16.
+pub fn constructor_icmps_simm16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: i16,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2861.
+    let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rsimm16(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmps_simm32.
+pub fn constructor_icmps_simm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: i32,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2864.
+    let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rsimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmps_mem.
+pub fn constructor_icmps_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2867.
+    let expr0_0 = constructor_cmpop_cmps(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmps_mem_sext16.
+pub fn constructor_icmps_mem_sext16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2870.
+    let expr0_0 = constructor_cmpop_cmps_sext16(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmps_mem_sext32.
+pub fn constructor_icmps_mem_sext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2873.
+    let expr0_0 = constructor_cmpop_cmps_sext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term cmpop_cmpu.
+pub fn constructor_cmpop_cmpu<C: Context>(ctx: &mut C, arg0: Type) -> Option<CmpOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2879.
+        let expr0_0 = CmpOp::CmpL32;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2880.
+        let expr0_0 = CmpOp::CmpL64;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term cmpop_cmpu_zext16.
+pub fn constructor_cmpop_cmpu_zext16<C: Context>(ctx: &mut C, arg0: Type) -> Option<CmpOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/inst.isle line 2883.
+        let expr0_0 = CmpOp::CmpL32Ext16;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2884.
+        let expr0_0 = CmpOp::CmpL64Ext16;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term cmpop_cmpu_zext32.
+pub fn constructor_cmpop_cmpu_zext32<C: Context>(ctx: &mut C, arg0: Type) -> Option<CmpOp> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/inst.isle line 2887.
+        let expr0_0 = CmpOp::CmpL64Ext32;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term icmpu_reg.
+pub fn constructor_icmpu_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2890.
+    let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmpu_reg_zext32.
+pub fn constructor_icmpu_reg_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2893.
+    let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rr(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmpu_uimm32.
+pub fn constructor_icmpu_uimm32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: u32,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2896.
+    let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_ruimm32(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmpu_mem.
+pub fn constructor_icmpu_mem<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2899.
+    let expr0_0 = constructor_cmpop_cmpu(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmpu_mem_zext16.
+pub fn constructor_icmpu_mem_zext16<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2902.
+    let expr0_0 = constructor_cmpop_cmpu_zext16(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term icmpu_mem_zext32.
+pub fn constructor_icmpu_mem_zext32<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: &MemArg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = arg2;
+    // Rule at src/isa/s390x/inst.isle line 2905.
+    let expr0_0 = constructor_cmpop_cmpu_zext32(ctx, pattern0_0)?;
+    let expr1_0 = constructor_cmp_rx(ctx, &expr0_0, pattern1_0, pattern2_0)?;
+    return Some(expr1_0);
+}
+
+// Generated as internal constructor for term fcmp_reg.
+pub fn constructor_fcmp_reg<C: Context>(
+    ctx: &mut C,
+    arg0: Type,
+    arg1: Reg,
+    arg2: Reg,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == F32 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2911.
+        let expr0_0 = constructor_fpu_cmp32(ctx, pattern2_0, pattern3_0)?;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == F64 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/inst.isle line 2912.
+        let expr0_0 = constructor_fpu_cmp64(ctx, pattern2_0, pattern3_0)?;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term lower.
+pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = C::inst_data(ctx, pattern0_0);
+    match &pattern1_0 {
+        &InstructionData::NullAry {
+            opcode: ref pattern2_0,
+        } => {
+            match &pattern2_0 {
+                &Opcode::Nop => {
+                    // Rule at src/isa/s390x/lower.isle line 42.
+                    let expr0_0 = C::invalid_reg(ctx);
+                    let expr1_0 = C::value_reg(ctx, expr0_0);
+                    return Some(expr1_0);
+                }
+                &Opcode::Fence => {
+                    // Rule at src/isa/s390x/lower.isle line 1600.
+                    let expr0_0 = constructor_fence_impl(ctx)?;
+                    let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                    return Some(expr1_0);
+                }
+                _ => {}
+            }
+        }
+        &InstructionData::FuncAddr {
+            opcode: ref pattern2_0,
+            func_ref: pattern2_1,
+        } => {
+            if let &Opcode::FuncAddr = &pattern2_0 {
+                if let Some((pattern4_0, pattern4_1)) = C::call_target_data(ctx, pattern0_0) {
+                    if let Some(()) = C::reloc_distance_near(ctx, &pattern4_1) {
+                        // Rule at src/isa/s390x/lower.isle line 1154.
+                        let expr0_0: i32 = 0;
+                        let expr1_0 = C::memflags_trusted(ctx);
+                        let expr2_0 = C::memarg_symbol(ctx, pattern4_0, expr0_0, expr1_0);
+                        let expr3_0 = constructor_load_addr(ctx, &expr2_0)?;
+                        let expr4_0 = C::value_reg(ctx, expr3_0);
+                        return Some(expr4_0);
+                    }
+                    // Rule at src/isa/s390x/lower.isle line 1159.
+                    let expr0_0: i64 = 0;
+                    let expr1_0 = constructor_load_ext_name_far(ctx, pattern4_0, expr0_0)?;
+                    let expr2_0 = C::value_reg(ctx, expr1_0);
+                    return Some(expr2_0);
+                }
+            }
+        }
+        &InstructionData::UnaryGlobalValue {
+            opcode: ref pattern2_0,
+            global_value: pattern2_1,
+        } => {
+            if let &Opcode::SymbolValue = &pattern2_0 {
+                if let Some((pattern4_0, pattern4_1, pattern4_2)) =
+                    C::symbol_value_data(ctx, pattern0_0)
+                {
+                    if let Some(()) = C::reloc_distance_near(ctx, &pattern4_1) {
+                        let pattern6_0 = 0;
+                        if let Some(pattern7_0) =
+                            C::memarg_symbol_offset_sum(ctx, pattern4_2, pattern6_0)
+                        {
+                            // Rule at src/isa/s390x/lower.isle line 1167.
+                            let expr0_0 = C::memflags_trusted(ctx);
+                            let expr1_0 = C::memarg_symbol(ctx, pattern4_0, pattern7_0, expr0_0);
+                            let expr2_0 = constructor_load_addr(ctx, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                    }
+                    // Rule at src/isa/s390x/lower.isle line 1173.
+                    let expr0_0 = constructor_load_ext_name_far(ctx, pattern4_0, pattern4_2)?;
+                    let expr1_0 = C::value_reg(ctx, expr0_0);
+                    return Some(expr1_0);
+                }
+            }
+        }
+        &InstructionData::UnaryIeee32 {
+            opcode: ref pattern2_0,
+            imm: pattern2_1,
+        } => {
+            if let &Opcode::F32const = &pattern2_0 {
+                let pattern4_0 = C::u64_from_ieee32(ctx, pattern2_1);
+                // Rule at src/isa/s390x/lower.isle line 24.
+                let expr0_0: Type = F32;
+                let expr1_0 = constructor_imm(ctx, expr0_0, pattern4_0)?;
+                let expr2_0 = C::value_reg(ctx, expr1_0);
+                return Some(expr2_0);
+            }
+        }
+        &InstructionData::UnaryIeee64 {
+            opcode: ref pattern2_0,
+            imm: pattern2_1,
+        } => {
+            if let &Opcode::F64const = &pattern2_0 {
+                let pattern4_0 = C::u64_from_ieee64(ctx, pattern2_1);
+                // Rule at src/isa/s390x/lower.isle line 30.
+                let expr0_0: Type = F64;
+                let expr1_0 = constructor_imm(ctx, expr0_0, pattern4_0)?;
+                let expr2_0 = C::value_reg(ctx, expr1_0);
+                return Some(expr2_0);
+            }
+        }
+        &InstructionData::StoreNoOffset {
+            opcode: ref pattern2_0,
+            args: ref pattern2_1,
+            flags: pattern2_2,
+        } => {
+            if let &Opcode::AtomicStore = &pattern2_0 {
+                let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, &pattern2_1);
+                let pattern5_0 = C::value_type(ctx, pattern4_0);
+                if pattern5_0 == I8 {
+                    // Rule at src/isa/s390x/lower.isle line 1581.
+                    let expr0_0 = C::zero_offset(ctx);
+                    let expr1_0 =
+                        constructor_istore8_impl(ctx, pattern2_2, pattern4_0, pattern4_1, expr0_0)?;
+                    let expr2_0 = constructor_atomic_store_impl(ctx, &expr1_0)?;
+                    return Some(expr2_0);
+                }
+                if pattern5_0 == I16 {
+                    // Rule at src/isa/s390x/lower.isle line 1585.
+                    let expr0_0 = C::zero_offset(ctx);
+                    let expr1_0 = constructor_istore16_impl(
+                        ctx, pattern2_2, pattern4_0, pattern4_1, expr0_0,
+                    )?;
+                    let expr2_0 = constructor_atomic_store_impl(ctx, &expr1_0)?;
+                    return Some(expr2_0);
+                }
+                if pattern5_0 == I32 {
+                    // Rule at src/isa/s390x/lower.isle line 1589.
+                    let expr0_0 = C::zero_offset(ctx);
+                    let expr1_0 = constructor_istore32_impl(
+                        ctx, pattern2_2, pattern4_0, pattern4_1, expr0_0,
+                    )?;
+                    let expr2_0 = constructor_atomic_store_impl(ctx, &expr1_0)?;
+                    return Some(expr2_0);
+                }
+                if pattern5_0 == I64 {
+                    // Rule at src/isa/s390x/lower.isle line 1593.
+                    let expr0_0 = C::zero_offset(ctx);
+                    let expr1_0 = constructor_istore64_impl(
+                        ctx, pattern2_2, pattern4_0, pattern4_1, expr0_0,
+                    )?;
+                    let expr2_0 = constructor_atomic_store_impl(ctx, &expr1_0)?;
+                    return Some(expr2_0);
+                }
+            }
+        }
+        &InstructionData::Store {
+            opcode: ref pattern2_0,
+            args: ref pattern2_1,
+            flags: pattern2_2,
+            offset: pattern2_3,
+        } => {
+            match &pattern2_0 {
+                &Opcode::Store => {
+                    let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, &pattern2_1);
+                    let pattern5_0 = C::value_type(ctx, pattern4_0);
+                    if pattern5_0 == I8 {
+                        // Rule at src/isa/s390x/lower.isle line 1353.
+                        let expr0_0 = constructor_istore8_impl(
+                            ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                        )?;
+                        let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                    if pattern5_0 == I16 {
+                        // Rule at src/isa/s390x/lower.isle line 1357.
+                        let expr0_0 = constructor_istore16_impl(
+                            ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                        )?;
+                        let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                    if pattern5_0 == I32 {
+                        // Rule at src/isa/s390x/lower.isle line 1361.
+                        let expr0_0 = constructor_istore32_impl(
+                            ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                        )?;
+                        let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                    if pattern5_0 == I64 {
+                        // Rule at src/isa/s390x/lower.isle line 1365.
+                        let expr0_0 = constructor_istore64_impl(
+                            ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                        )?;
+                        let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                    if pattern5_0 == R64 {
+                        // Rule at src/isa/s390x/lower.isle line 1369.
+                        let expr0_0 = constructor_istore64_impl(
+                            ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                        )?;
+                        let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                        return Some(expr1_0);
+                    }
+                    if pattern5_0 == F32 {
+                        if let Some(()) = C::bigendian(ctx, pattern2_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1373.
+                            let expr0_0 = C::put_in_reg(ctx, pattern4_0);
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern2_2, pattern4_1, pattern2_3)?;
+                            let expr2_0 = constructor_fpu_store32(ctx, expr0_0, &expr1_0)?;
+                            let expr3_0 = constructor_value_regs_none(ctx, &expr2_0)?;
+                            return Some(expr3_0);
+                        }
+                        if let Some(()) = C::vxrs_ext2_enabled(ctx, pattern5_0) {
+                            if let Some(()) = C::littleendian(ctx, pattern2_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1379.
+                                let expr0_0 = C::put_in_reg(ctx, pattern4_0);
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern2_2, pattern4_1, pattern2_3,
+                                )?;
+                                let expr2_0 = constructor_fpu_storerev32(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = constructor_value_regs_none(ctx, &expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                        }
+                        if let Some(()) = C::vxrs_ext2_disabled(ctx, pattern5_0) {
+                            if let Some(()) = C::littleendian(ctx, pattern2_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1385.
+                                let expr0_0: Type = I64;
+                                let expr1_0 = C::put_in_reg(ctx, pattern4_0);
+                                let expr2_0 = constructor_mov_from_fpr(ctx, expr1_0)?;
+                                let expr3_0: u8 = 32;
+                                let expr4_0 = constructor_lshr_imm(ctx, expr0_0, expr2_0, expr3_0)?;
+                                let expr5_0 = constructor_lower_address(
+                                    ctx, pattern2_2, pattern4_1, pattern2_3,
+                                )?;
+                                let expr6_0 = constructor_storerev32(ctx, expr4_0, &expr5_0)?;
+                                let expr7_0 = constructor_value_regs_none(ctx, &expr6_0)?;
+                                return Some(expr7_0);
+                            }
+                        }
+                    }
+                    if pattern5_0 == F64 {
+                        if let Some(()) = C::bigendian(ctx, pattern2_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1391.
+                            let expr0_0 = C::put_in_reg(ctx, pattern4_0);
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern2_2, pattern4_1, pattern2_3)?;
+                            let expr2_0 = constructor_fpu_store64(ctx, expr0_0, &expr1_0)?;
+                            let expr3_0 = constructor_value_regs_none(ctx, &expr2_0)?;
+                            return Some(expr3_0);
+                        }
+                        if let Some(()) = C::vxrs_ext2_enabled(ctx, pattern5_0) {
+                            if let Some(()) = C::littleendian(ctx, pattern2_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1397.
+                                let expr0_0 = C::put_in_reg(ctx, pattern4_0);
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern2_2, pattern4_1, pattern2_3,
+                                )?;
+                                let expr2_0 = constructor_fpu_storerev64(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = constructor_value_regs_none(ctx, &expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                        }
+                        if let Some(()) = C::vxrs_ext2_disabled(ctx, pattern5_0) {
+                            if let Some(()) = C::littleendian(ctx, pattern2_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1403.
+                                let expr0_0 = C::put_in_reg(ctx, pattern4_0);
+                                let expr1_0 = constructor_mov_from_fpr(ctx, expr0_0)?;
+                                let expr2_0 = constructor_lower_address(
+                                    ctx, pattern2_2, pattern4_1, pattern2_3,
+                                )?;
+                                let expr3_0 = constructor_storerev64(ctx, expr1_0, &expr2_0)?;
+                                let expr4_0 = constructor_value_regs_none(ctx, &expr3_0)?;
+                                return Some(expr4_0);
+                            }
+                        }
+                    }
+                }
+                &Opcode::Istore8 => {
+                    let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, &pattern2_1);
+                    // Rule at src/isa/s390x/lower.isle line 1412.
+                    let expr0_0 = constructor_istore8_impl(
+                        ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                    )?;
+                    let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                    return Some(expr1_0);
+                }
+                &Opcode::Istore16 => {
+                    let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, &pattern2_1);
+                    // Rule at src/isa/s390x/lower.isle line 1430.
+                    let expr0_0 = constructor_istore16_impl(
+                        ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                    )?;
+                    let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                    return Some(expr1_0);
+                }
+                &Opcode::Istore32 => {
+                    let (pattern4_0, pattern4_1) = C::unpack_value_array_2(ctx, &pattern2_1);
+                    // Rule at src/isa/s390x/lower.isle line 1456.
+                    let expr0_0 = constructor_istore32_impl(
+                        ctx, pattern2_2, pattern4_0, pattern4_1, pattern2_3,
+                    )?;
+                    let expr1_0 = constructor_value_regs_none(ctx, &expr0_0)?;
+                    return Some(expr1_0);
+                }
+                _ => {}
+            }
+        }
+        &InstructionData::Unary {
+            opcode: ref pattern2_0,
+            arg: pattern2_1,
+        } => {
+            match &pattern2_0 {
+                &Opcode::Copy => {
+                    // Rule at src/isa/s390x/lower.isle line 48.
+                    let expr0_0 = C::put_in_reg(ctx, pattern2_1);
+                    let expr1_0 = C::value_reg(ctx, expr0_0);
+                    return Some(expr1_0);
+                }
+                &Opcode::Breduce => {
+                    // Rule at src/isa/s390x/lower.isle line 747.
+                    let expr0_0 = C::put_in_reg(ctx, pattern2_1);
+                    let expr1_0 = C::value_reg(ctx, expr0_0);
+                    return Some(expr1_0);
+                }
+                &Opcode::Ireduce => {
+                    // Rule at src/isa/s390x/lower.isle line 591.
+                    let expr0_0 = C::put_in_reg(ctx, pattern2_1);
+                    let expr1_0 = C::value_reg(ctx, expr0_0);
+                    return Some(expr1_0);
+                }
+                _ => {}
+            }
+        }
+        _ => {}
+    }
+    if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) {
+        let pattern2_0 = C::value_type(ctx, pattern1_0);
+        if pattern2_0 == B1 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Unary {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+            } = &pattern4_0
+            {
+                match &pattern5_0 {
+                    &Opcode::IsNull => {
+                        let pattern7_0 = C::value_type(ctx, pattern5_1);
+                        if pattern7_0 == R64 {
+                            // Rule at src/isa/s390x/lower.isle line 1735.
+                            let expr0_0: Type = B1;
+                            let expr1_0: Type = I64;
+                            let expr2_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr3_0: i16 = 0;
+                            let expr4_0 = constructor_icmps_simm16(ctx, expr1_0, expr2_0, expr3_0)?;
+                            let expr5_0 = IntCC::Equal;
+                            let expr6_0 = C::intcc_as_cond(ctx, &expr5_0);
+                            let expr7_0 = constructor_bool(ctx, &expr4_0, &expr6_0)?;
+                            let expr8_0 = constructor_lower_bool(ctx, expr0_0, &expr7_0)?;
+                            let expr9_0 = C::value_reg(ctx, expr8_0);
+                            return Some(expr9_0);
+                        }
+                    }
+                    &Opcode::IsInvalid => {
+                        let pattern7_0 = C::value_type(ctx, pattern5_1);
+                        if pattern7_0 == R64 {
+                            // Rule at src/isa/s390x/lower.isle line 1741.
+                            let expr0_0: Type = B1;
+                            let expr1_0: Type = I64;
+                            let expr2_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr3_0: i16 = -1;
+                            let expr4_0 = constructor_icmps_simm16(ctx, expr1_0, expr2_0, expr3_0)?;
+                            let expr5_0 = IntCC::Equal;
+                            let expr6_0 = C::intcc_as_cond(ctx, &expr5_0);
+                            let expr7_0 = constructor_bool(ctx, &expr4_0, &expr6_0)?;
+                            let expr8_0 = constructor_lower_bool(ctx, expr0_0, &expr7_0)?;
+                            let expr9_0 = C::value_reg(ctx, expr8_0);
+                            return Some(expr9_0);
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
+        if pattern2_0 == I8 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    if let &Opcode::Popcnt = &pattern5_0 {
+                        // Rule at src/isa/s390x/lower.isle line 879.
+                        let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                        let expr1_0 = constructor_popcnt_byte(ctx, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                }
+                &InstructionData::LoadNoOffset {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                } => {
+                    if let &Opcode::AtomicLoad = &pattern5_0 {
+                        // Rule at src/isa/s390x/lower.isle line 1544.
+                        let expr0_0: Type = I8;
+                        let expr1_0 = C::zero_offset(ctx);
+                        let expr2_0 =
+                            constructor_lower_address(ctx, pattern5_2, pattern5_1, expr1_0)?;
+                        let expr3_0 = constructor_zext32_mem(ctx, expr0_0, &expr2_0)?;
+                        let expr4_0 = C::value_reg(ctx, expr3_0);
+                        return Some(expr4_0);
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    if let &Opcode::Load = &pattern5_0 {
+                        // Rule at src/isa/s390x/lower.isle line 1181.
+                        let expr0_0: Type = I8;
+                        let expr1_0 =
+                            constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                        let expr2_0 = constructor_zext32_mem(ctx, expr0_0, &expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                }
+                _ => {}
+            }
+        }
+        if pattern2_0 == I16 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::LoadNoOffset {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                } => {
+                    if let &Opcode::AtomicLoad = &pattern5_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1552.
+                            let expr0_0 = C::zero_offset(ctx);
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, expr0_0)?;
+                            let expr2_0 = constructor_loadrev16(ctx, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1548.
+                            let expr0_0: Type = I16;
+                            let expr1_0 = C::zero_offset(ctx);
+                            let expr2_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, expr1_0)?;
+                            let expr3_0 = constructor_zext32_mem(ctx, expr0_0, &expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    if let &Opcode::Load = &pattern5_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1189.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr1_0 = constructor_loadrev16(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1185.
+                            let expr0_0: Type = I16;
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr2_0 = constructor_zext32_mem(ctx, expr0_0, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+        if pattern2_0 == I32 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Binary {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Umulhi => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 247.
+                            let expr0_0 = constructor_put_in_reg_zext64(ctx, pattern7_0)?;
+                            let expr1_0 = constructor_put_in_reg_zext64(ctx, pattern7_1)?;
+                            let expr2_0: Type = I64;
+                            let expr3_0 = constructor_mul_reg(ctx, expr2_0, expr0_0, expr1_0)?;
+                            let expr4_0: Type = I64;
+                            let expr5_0: u8 = 32;
+                            let expr6_0 = constructor_lshr_imm(ctx, expr4_0, expr3_0, expr5_0)?;
+                            let expr7_0 = C::value_reg(ctx, expr6_0);
+                            return Some(expr7_0);
+                        }
+                        &Opcode::Smulhi => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 269.
+                            let expr0_0 = constructor_put_in_reg_sext64(ctx, pattern7_0)?;
+                            let expr1_0 = constructor_put_in_reg_sext64(ctx, pattern7_1)?;
+                            let expr2_0: Type = I64;
+                            let expr3_0 = constructor_mul_reg(ctx, expr2_0, expr0_0, expr1_0)?;
+                            let expr4_0: Type = I64;
+                            let expr5_0: u8 = 32;
+                            let expr6_0 = constructor_ashr_imm(ctx, expr4_0, expr3_0, expr5_0)?;
+                            let expr7_0 = C::value_reg(ctx, expr6_0);
+                            return Some(expr7_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::AtomicCas {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                    flags: pattern5_2,
+                } => {
+                    if let &Opcode::AtomicCas = &pattern5_0 {
+                        let (pattern7_0, pattern7_1, pattern7_2) =
+                            C::unpack_value_array_3(ctx, &pattern5_1);
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1529.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_2);
+                            let expr2_0 = C::zero_offset(ctx);
+                            let expr3_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern7_0, expr2_0)?;
+                            let expr4_0 =
+                                constructor_atomic_cas32(ctx, expr0_0, expr1_0, &expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                    }
+                }
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    if let &Opcode::Bitcast = &pattern5_0 {
+                        let pattern7_0 = C::value_type(ctx, pattern5_1);
+                        if pattern7_0 == F32 {
+                            // Rule at src/isa/s390x/lower.isle line 1140.
+                            let expr0_0: Type = I64;
+                            let expr1_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr2_0 = constructor_mov_from_fpr(ctx, expr1_0)?;
+                            let expr3_0: u8 = 32;
+                            let expr4_0 = constructor_lshr_imm(ctx, expr0_0, expr2_0, expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                    }
+                }
+                &InstructionData::LoadNoOffset {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                } => {
+                    if let &Opcode::AtomicLoad = &pattern5_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1560.
+                            let expr0_0 = C::zero_offset(ctx);
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, expr0_0)?;
+                            let expr2_0 = constructor_loadrev32(ctx, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1556.
+                            let expr0_0 = C::zero_offset(ctx);
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, expr0_0)?;
+                            let expr2_0 = constructor_load32(ctx, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    if let &Opcode::Load = &pattern5_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1197.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr1_0 = constructor_loadrev32(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1193.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr1_0 = constructor_load32(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+        if pattern2_0 == I64 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Binary {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Umulhi => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 254.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_umul_wide(ctx, expr0_0, expr1_0)?;
+                            let expr3_0: Type = I64;
+                            let expr4_0 = constructor_regpair_hi(ctx, &expr2_0)?;
+                            let expr5_0 = constructor_copy_reg(ctx, expr3_0, expr4_0)?;
+                            let expr6_0 = C::value_reg(ctx, expr5_0);
+                            return Some(expr6_0);
+                        }
+                        &Opcode::Smulhi => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 276.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_smul_wide(ctx, expr0_0, expr1_0)?;
+                            let expr3_0: Type = I64;
+                            let expr4_0 = constructor_regpair_hi(ctx, &expr2_0)?;
+                            let expr5_0 = constructor_copy_reg(ctx, expr3_0, expr4_0)?;
+                            let expr6_0 = C::value_reg(ctx, expr5_0);
+                            return Some(expr6_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::AtomicCas {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                    flags: pattern5_2,
+                } => {
+                    if let &Opcode::AtomicCas = &pattern5_0 {
+                        let (pattern7_0, pattern7_1, pattern7_2) =
+                            C::unpack_value_array_3(ctx, &pattern5_1);
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1534.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_2);
+                            let expr2_0 = C::zero_offset(ctx);
+                            let expr3_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern7_0, expr2_0)?;
+                            let expr4_0 =
+                                constructor_atomic_cas64(ctx, expr0_0, expr1_0, &expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                    }
+                }
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    if let &Opcode::Bitcast = &pattern5_0 {
+                        let pattern7_0 = C::value_type(ctx, pattern5_1);
+                        if pattern7_0 == F64 {
+                            // Rule at src/isa/s390x/lower.isle line 1130.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0 = constructor_mov_from_fpr(ctx, expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+                &InstructionData::LoadNoOffset {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                } => {
+                    if let &Opcode::AtomicLoad = &pattern5_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1568.
+                            let expr0_0 = C::zero_offset(ctx);
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, expr0_0)?;
+                            let expr2_0 = constructor_loadrev64(ctx, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1564.
+                            let expr0_0 = C::zero_offset(ctx);
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, expr0_0)?;
+                            let expr2_0 = constructor_load64(ctx, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    if let &Opcode::Load = &pattern5_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1205.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr1_0 = constructor_loadrev64(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1201.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr1_0 = constructor_load64(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+        if pattern2_0 == R64 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Load {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+                flags: pattern5_2,
+                offset: pattern5_3,
+            } = &pattern4_0
+            {
+                if let &Opcode::Load = &pattern5_0 {
+                    if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                        // Rule at src/isa/s390x/lower.isle line 1213.
+                        let expr0_0 =
+                            constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                        let expr1_0 = constructor_loadrev64(ctx, &expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                        // Rule at src/isa/s390x/lower.isle line 1209.
+                        let expr0_0 =
+                            constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                        let expr1_0 = constructor_load64(ctx, &expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                }
+            }
+        }
+        if pattern2_0 == F32 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    if let &Opcode::Bitcast = &pattern5_0 {
+                        let pattern7_0 = C::value_type(ctx, pattern5_1);
+                        if pattern7_0 == I32 {
+                            // Rule at src/isa/s390x/lower.isle line 1135.
+                            let expr0_0: Type = I64;
+                            let expr1_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr2_0: u8 = 32;
+                            let expr3_0 = constructor_lshl_imm(ctx, expr0_0, expr1_0, expr2_0)?;
+                            let expr4_0 = constructor_mov_to_fpr(ctx, expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    if let &Opcode::Load = &pattern5_0 {
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1217.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr1_0 = constructor_fpu_load32(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+        if pattern2_0 == F64 {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    if let &Opcode::Bitcast = &pattern5_0 {
+                        let pattern7_0 = C::value_type(ctx, pattern5_1);
+                        if pattern7_0 == I64 {
+                            // Rule at src/isa/s390x/lower.isle line 1126.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0 = constructor_mov_to_fpr(ctx, expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    if let &Opcode::Load = &pattern5_0 {
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1232.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr1_0 = constructor_fpu_load64(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+        let pattern3_0 = C::inst_data(ctx, pattern0_0);
+        match &pattern3_0 {
+            &InstructionData::NullAry {
+                opcode: ref pattern4_0,
+            } => {
+                if let &Opcode::Null = &pattern4_0 {
+                    // Rule at src/isa/s390x/lower.isle line 36.
+                    let expr0_0: u64 = 0;
+                    let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?;
+                    let expr2_0 = C::value_reg(ctx, expr1_0);
+                    return Some(expr2_0);
+                }
+            }
+            &InstructionData::UnaryImm {
+                opcode: ref pattern4_0,
+                imm: pattern4_1,
+            } => {
+                if let &Opcode::Iconst = &pattern4_0 {
+                    let pattern6_0 = C::u64_from_imm64(ctx, pattern4_1);
+                    // Rule at src/isa/s390x/lower.isle line 10.
+                    let expr0_0 = constructor_imm(ctx, pattern2_0, pattern6_0)?;
+                    let expr1_0 = C::value_reg(ctx, expr0_0);
+                    return Some(expr1_0);
+                }
+            }
+            &InstructionData::StackLoad {
+                opcode: ref pattern4_0,
+                stack_slot: pattern4_1,
+                offset: pattern4_2,
+            } => {
+                if let &Opcode::StackAddr = &pattern4_0 {
+                    // Rule at src/isa/s390x/lower.isle line 1147.
+                    let expr0_0 =
+                        constructor_stack_addr_impl(ctx, pattern2_0, pattern4_1, pattern4_2)?;
+                    let expr1_0 = C::value_reg(ctx, expr0_0);
+                    return Some(expr1_0);
+                }
+            }
+            &InstructionData::UnaryBool {
+                opcode: ref pattern4_0,
+                imm: pattern4_1,
+            } => {
+                if let &Opcode::Bconst = &pattern4_0 {
+                    if pattern4_1 == true {
+                        // Rule at src/isa/s390x/lower.isle line 18.
+                        let expr0_0: u64 = 1;
+                        let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    if pattern4_1 == false {
+                        // Rule at src/isa/s390x/lower.isle line 16.
+                        let expr0_0: u64 = 0;
+                        let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                }
+            }
+            &InstructionData::Binary {
+                opcode: ref pattern4_0,
+                args: ref pattern4_1,
+            } => {
+                match &pattern4_0 {
+                    &Opcode::Fadd => {
+                        let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 915.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = constructor_fadd_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Fsub => {
+                        let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 922.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = constructor_fsub_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Fmul => {
+                        let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 929.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = constructor_fmul_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Fdiv => {
+                        let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 936.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = constructor_fdiv_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Fcopysign => {
+                        let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 957.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = constructor_fpu_copysign(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Fmin => {
+                        let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 943.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = constructor_fmin_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::Fmax => {
+                        let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 950.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = constructor_fmax_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    _ => {}
+                }
+            }
+            &InstructionData::FloatCompare {
+                opcode: ref pattern4_0,
+                args: ref pattern4_1,
+                cond: ref pattern4_2,
+            } => {
+                if let &Opcode::Fcmp = &pattern4_0 {
+                    let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                    // Rule at src/isa/s390x/lower.isle line 1722.
+                    let expr0_0 = constructor_fcmp_val(ctx, &pattern4_2, pattern6_0, pattern6_1)?;
+                    let expr1_0 = constructor_lower_bool(ctx, pattern2_0, &expr0_0)?;
+                    let expr2_0 = C::value_reg(ctx, expr1_0);
+                    return Some(expr2_0);
+                }
+            }
+            &InstructionData::IntCompare {
+                opcode: ref pattern4_0,
+                args: ref pattern4_1,
+                cond: ref pattern4_2,
+            } => {
+                if let &Opcode::Icmp = &pattern4_0 {
+                    let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1);
+                    // Rule at src/isa/s390x/lower.isle line 1633.
+                    let expr0_0: bool = true;
+                    let expr1_0 =
+                        constructor_icmp_val(ctx, expr0_0, &pattern4_2, pattern6_0, pattern6_1)?;
+                    let expr2_0 = constructor_lower_bool(ctx, pattern2_0, &expr1_0)?;
+                    let expr3_0 = C::value_reg(ctx, expr2_0);
+                    return Some(expr3_0);
+                }
+            }
+            &InstructionData::Ternary {
+                opcode: ref pattern4_0,
+                args: ref pattern4_1,
+            } => {
+                match &pattern4_0 {
+                    &Opcode::Select => {
+                        let (pattern6_0, pattern6_1, pattern6_2) =
+                            C::unpack_value_array_3(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 1764.
+                        let expr0_0 = constructor_value_nonzero(ctx, pattern6_0)?;
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = C::put_in_reg(ctx, pattern6_2);
+                        let expr3_0 = constructor_select_bool_reg(
+                            ctx, pattern2_0, &expr0_0, expr1_0, expr2_0,
+                        )?;
+                        let expr4_0 = C::value_reg(ctx, expr3_0);
+                        return Some(expr4_0);
+                    }
+                    &Opcode::Fma => {
+                        let (pattern6_0, pattern6_1, pattern6_2) =
+                            C::unpack_value_array_3(ctx, &pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 964.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr2_0 = C::put_in_reg(ctx, pattern6_2);
+                        let expr3_0 =
+                            constructor_fma_reg(ctx, pattern2_0, expr0_0, expr1_0, expr2_0)?;
+                        let expr4_0 = C::value_reg(ctx, expr3_0);
+                        return Some(expr4_0);
+                    }
+                    _ => {}
+                }
+            }
+            &InstructionData::IntSelect {
+                opcode: ref pattern4_0,
+                args: ref pattern4_1,
+                cond: ref pattern4_2,
+            } => {
+                if let &Opcode::SelectifSpectreGuard = &pattern4_0 {
+                    let (pattern6_0, pattern6_1, pattern6_2) =
+                        C::unpack_value_array_3(ctx, &pattern4_1);
+                    if let Some(pattern7_0) = C::def_inst(ctx, pattern6_0) {
+                        let pattern8_0 = C::inst_data(ctx, pattern7_0);
+                        if let &InstructionData::Binary {
+                            opcode: ref pattern9_0,
+                            args: ref pattern9_1,
+                        } = &pattern8_0
+                        {
+                            if let &Opcode::Ifcmp = &pattern9_0 {
+                                let (pattern11_0, pattern11_1) =
+                                    C::unpack_value_array_2(ctx, &pattern9_1);
+                                // Rule at src/isa/s390x/lower.isle line 1776.
+                                let expr0_0: bool = false;
+                                let expr1_0 = constructor_icmp_val(
+                                    ctx,
+                                    expr0_0,
+                                    &pattern4_2,
+                                    pattern11_0,
+                                    pattern11_1,
+                                )?;
+                                let expr2_0 = C::put_in_reg(ctx, pattern6_1);
+                                let expr3_0 = C::put_in_reg(ctx, pattern6_2);
+                                let expr4_0 = constructor_select_bool_reg(
+                                    ctx, pattern2_0, &expr1_0, expr2_0, expr3_0,
+                                )?;
+                                let expr5_0 = C::value_reg(ctx, expr4_0);
+                                return Some(expr5_0);
+                            }
+                        }
+                    }
+                }
+            }
+            &InstructionData::Unary {
+                opcode: ref pattern4_0,
+                arg: pattern4_1,
+            } => {
+                match &pattern4_0 {
+                    &Opcode::Sqrt => {
+                        // Rule at src/isa/s390x/lower.isle line 971.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 = constructor_sqrt_reg(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Fneg => {
+                        // Rule at src/isa/s390x/lower.isle line 978.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 = constructor_fneg_reg(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Fabs => {
+                        // Rule at src/isa/s390x/lower.isle line 985.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 = constructor_fabs_reg(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Ceil => {
+                        // Rule at src/isa/s390x/lower.isle line 992.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 = constructor_ceil_reg(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Floor => {
+                        // Rule at src/isa/s390x/lower.isle line 999.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 = constructor_floor_reg(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Trunc => {
+                        // Rule at src/isa/s390x/lower.isle line 1006.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 = constructor_trunc_reg(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Nearest => {
+                        // Rule at src/isa/s390x/lower.isle line 1013.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 = constructor_nearest_reg(ctx, pattern2_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Bextend => {
+                        // Rule at src/isa/s390x/lower.isle line 755.
+                        let expr0_0 = constructor_cast_bool(ctx, pattern2_0, pattern4_1)?;
+                        let expr1_0 = C::value_reg(ctx, expr0_0);
+                        return Some(expr1_0);
+                    }
+                    &Opcode::Bmask => {
+                        // Rule at src/isa/s390x/lower.isle line 757.
+                        let expr0_0 = constructor_cast_bool(ctx, pattern2_0, pattern4_1)?;
+                        let expr1_0 = C::value_reg(ctx, expr0_0);
+                        return Some(expr1_0);
+                    }
+                    &Opcode::Fpromote => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 1020.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 =
+                            constructor_fpromote_reg(ctx, pattern2_0, pattern6_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::Fdemote => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 1027.
+                        let expr0_0 = C::put_in_reg(ctx, pattern4_1);
+                        let expr1_0 =
+                            constructor_fdemote_reg(ctx, pattern2_0, pattern6_0, expr0_0)?;
+                        let expr2_0 = C::value_reg(ctx, expr1_0);
+                        return Some(expr2_0);
+                    }
+                    &Opcode::FcvtFromUint => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 1034.
+                        let expr0_0 = constructor_ty_ext32(ctx, pattern6_0)?;
+                        let expr1_0 = constructor_put_in_reg_zext32(ctx, pattern4_1)?;
+                        let expr2_0 =
+                            constructor_fcvt_from_uint_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    &Opcode::FcvtFromSint => {
+                        let pattern6_0 = C::value_type(ctx, pattern4_1);
+                        // Rule at src/isa/s390x/lower.isle line 1042.
+                        let expr0_0 = constructor_ty_ext32(ctx, pattern6_0)?;
+                        let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern4_1)?;
+                        let expr2_0 =
+                            constructor_fcvt_from_sint_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+                        let expr3_0 = C::value_reg(ctx, expr2_0);
+                        return Some(expr3_0);
+                    }
+                    _ => {}
+                }
+            }
+            _ => {}
+        }
+        if let Some(()) = C::mie2_enabled(ctx, pattern2_0) {
+            if let Some(pattern4_0) = C::fits_in_64(ctx, pattern2_0) {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                match &pattern5_0 {
+                    &InstructionData::Binary {
+                        opcode: ref pattern6_0,
+                        args: ref pattern6_1,
+                    } => {
+                        match &pattern6_0 {
+                            &Opcode::BandNot => {
+                                let (pattern8_0, pattern8_1) =
+                                    C::unpack_value_array_2(ctx, &pattern6_1);
+                                // Rule at src/isa/s390x/lower.isle line 697.
+                                let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 =
+                                    constructor_and_not_reg(ctx, pattern4_0, expr0_0, expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::BorNot => {
+                                let (pattern8_0, pattern8_1) =
+                                    C::unpack_value_array_2(ctx, &pattern6_1);
+                                // Rule at src/isa/s390x/lower.isle line 708.
+                                let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 =
+                                    constructor_or_not_reg(ctx, pattern4_0, expr0_0, expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            &Opcode::BxorNot => {
+                                let (pattern8_0, pattern8_1) =
+                                    C::unpack_value_array_2(ctx, &pattern6_1);
+                                // Rule at src/isa/s390x/lower.isle line 719.
+                                let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 =
+                                    constructor_xor_not_reg(ctx, pattern4_0, expr0_0, expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                    &InstructionData::Ternary {
+                        opcode: ref pattern6_0,
+                        args: ref pattern6_1,
+                    } => {
+                        if let &Opcode::Bitselect = &pattern6_0 {
+                            let (pattern8_0, pattern8_1, pattern8_2) =
+                                C::unpack_value_array_3(ctx, &pattern6_1);
+                            // Rule at src/isa/s390x/lower.isle line 730.
+                            let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                            let expr2_0 = constructor_and_reg(ctx, pattern4_0, expr1_0, expr0_0)?;
+                            let expr3_0 = C::put_in_reg(ctx, pattern8_2);
+                            let expr4_0 =
+                                constructor_and_not_reg(ctx, pattern4_0, expr3_0, expr0_0)?;
+                            let expr5_0 = constructor_or_reg(ctx, pattern4_0, expr4_0, expr2_0)?;
+                            let expr6_0 = C::value_reg(ctx, expr5_0);
+                            return Some(expr6_0);
+                        }
+                    }
+                    &InstructionData::Unary {
+                        opcode: ref pattern6_0,
+                        arg: pattern6_1,
+                    } => {
+                        match &pattern6_0 {
+                            &Opcode::Bnot => {
+                                // Rule at src/isa/s390x/lower.isle line 620.
+                                let expr0_0 = C::put_in_reg(ctx, pattern6_1);
+                                let expr1_0 =
+                                    constructor_or_not_reg(ctx, pattern4_0, expr0_0, expr0_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            &Opcode::Popcnt => {
+                                // Rule at src/isa/s390x/lower.isle line 884.
+                                let expr0_0 = constructor_put_in_reg_zext64(ctx, pattern6_1)?;
+                                let expr1_0 = constructor_popcnt_reg(ctx, expr0_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
+        if let Some(()) = C::mie2_disabled(ctx, pattern2_0) {
+            if pattern2_0 == I16 {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::Unary {
+                    opcode: ref pattern6_0,
+                    arg: pattern6_1,
+                } = &pattern5_0
+                {
+                    if let &Opcode::Popcnt = &pattern6_0 {
+                        // Rule at src/isa/s390x/lower.isle line 893.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr1_0 = constructor_popcnt_byte(ctx, expr0_0)?;
+                        let expr2_0: Type = I32;
+                        let expr3_0: Type = I32;
+                        let expr4_0: u8 = 8;
+                        let expr5_0 = constructor_lshl_imm(ctx, expr3_0, expr1_0, expr4_0)?;
+                        let expr6_0 = constructor_add_reg(ctx, expr2_0, expr1_0, expr5_0)?;
+                        let expr7_0: Type = I32;
+                        let expr8_0: u8 = 8;
+                        let expr9_0 = constructor_lshr_imm(ctx, expr7_0, expr6_0, expr8_0)?;
+                        let expr10_0 = C::value_reg(ctx, expr9_0);
+                        return Some(expr10_0);
+                    }
+                }
+            }
+            if pattern2_0 == I32 {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::Unary {
+                    opcode: ref pattern6_0,
+                    arg: pattern6_1,
+                } = &pattern5_0
+                {
+                    if let &Opcode::Popcnt = &pattern6_0 {
+                        // Rule at src/isa/s390x/lower.isle line 898.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr1_0 = constructor_popcnt_byte(ctx, expr0_0)?;
+                        let expr2_0: Type = I32;
+                        let expr3_0: Type = I32;
+                        let expr4_0: u8 = 16;
+                        let expr5_0 = constructor_lshl_imm(ctx, expr3_0, expr1_0, expr4_0)?;
+                        let expr6_0 = constructor_add_reg(ctx, expr2_0, expr1_0, expr5_0)?;
+                        let expr7_0: Type = I32;
+                        let expr8_0: Type = I32;
+                        let expr9_0: u8 = 8;
+                        let expr10_0 = constructor_lshl_imm(ctx, expr8_0, expr6_0, expr9_0)?;
+                        let expr11_0 = constructor_add_reg(ctx, expr7_0, expr6_0, expr10_0)?;
+                        let expr12_0: Type = I32;
+                        let expr13_0: u8 = 24;
+                        let expr14_0 = constructor_lshr_imm(ctx, expr12_0, expr11_0, expr13_0)?;
+                        let expr15_0 = C::value_reg(ctx, expr14_0);
+                        return Some(expr15_0);
+                    }
+                }
+            }
+            if pattern2_0 == I64 {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::Unary {
+                    opcode: ref pattern6_0,
+                    arg: pattern6_1,
+                } = &pattern5_0
+                {
+                    if let &Opcode::Popcnt = &pattern6_0 {
+                        // Rule at src/isa/s390x/lower.isle line 904.
+                        let expr0_0 = C::put_in_reg(ctx, pattern6_1);
+                        let expr1_0 = constructor_popcnt_byte(ctx, expr0_0)?;
+                        let expr2_0: Type = I64;
+                        let expr3_0: Type = I64;
+                        let expr4_0: u8 = 32;
+                        let expr5_0 = constructor_lshl_imm(ctx, expr3_0, expr1_0, expr4_0)?;
+                        let expr6_0 = constructor_add_reg(ctx, expr2_0, expr1_0, expr5_0)?;
+                        let expr7_0: Type = I64;
+                        let expr8_0: Type = I64;
+                        let expr9_0: u8 = 16;
+                        let expr10_0 = constructor_lshl_imm(ctx, expr8_0, expr6_0, expr9_0)?;
+                        let expr11_0 = constructor_add_reg(ctx, expr7_0, expr6_0, expr10_0)?;
+                        let expr12_0: Type = I64;
+                        let expr13_0: Type = I64;
+                        let expr14_0: u8 = 8;
+                        let expr15_0 = constructor_lshl_imm(ctx, expr13_0, expr11_0, expr14_0)?;
+                        let expr16_0 = constructor_add_reg(ctx, expr12_0, expr11_0, expr15_0)?;
+                        let expr17_0: Type = I64;
+                        let expr18_0: u8 = 56;
+                        let expr19_0 = constructor_lshr_imm(ctx, expr17_0, expr16_0, expr18_0)?;
+                        let expr20_0 = C::value_reg(ctx, expr19_0);
+                        return Some(expr20_0);
+                    }
+                }
+            }
+            if let Some(pattern4_0) = C::fits_in_64(ctx, pattern2_0) {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                match &pattern5_0 {
+                    &InstructionData::Binary {
+                        opcode: ref pattern6_0,
+                        args: ref pattern6_1,
+                    } => {
+                        match &pattern6_0 {
+                            &Opcode::BandNot => {
+                                let (pattern8_0, pattern8_1) =
+                                    C::unpack_value_array_2(ctx, &pattern6_1);
+                                // Rule at src/isa/s390x/lower.isle line 701.
+                                let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 =
+                                    constructor_and_reg(ctx, pattern4_0, expr0_0, expr1_0)?;
+                                let expr3_0 = constructor_not_reg(ctx, pattern4_0, expr2_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            &Opcode::BorNot => {
+                                let (pattern8_0, pattern8_1) =
+                                    C::unpack_value_array_2(ctx, &pattern6_1);
+                                // Rule at src/isa/s390x/lower.isle line 712.
+                                let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 =
+                                    constructor_or_reg(ctx, pattern4_0, expr0_0, expr1_0)?;
+                                let expr3_0 = constructor_not_reg(ctx, pattern4_0, expr2_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            &Opcode::BxorNot => {
+                                let (pattern8_0, pattern8_1) =
+                                    C::unpack_value_array_2(ctx, &pattern6_1);
+                                // Rule at src/isa/s390x/lower.isle line 723.
+                                let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                                let expr2_0 =
+                                    constructor_xor_reg(ctx, pattern4_0, expr0_0, expr1_0)?;
+                                let expr3_0 = constructor_not_reg(ctx, pattern4_0, expr2_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            _ => {}
+                        }
+                    }
+                    &InstructionData::Ternary {
+                        opcode: ref pattern6_0,
+                        args: ref pattern6_1,
+                    } => {
+                        if let &Opcode::Bitselect = &pattern6_0 {
+                            let (pattern8_0, pattern8_1, pattern8_2) =
+                                C::unpack_value_array_3(ctx, &pattern6_1);
+                            // Rule at src/isa/s390x/lower.isle line 737.
+                            let expr0_0 = C::put_in_reg(ctx, pattern8_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern8_1);
+                            let expr2_0 = constructor_and_reg(ctx, pattern4_0, expr1_0, expr0_0)?;
+                            let expr3_0 = C::put_in_reg(ctx, pattern8_2);
+                            let expr4_0 = constructor_and_reg(ctx, pattern4_0, expr3_0, expr0_0)?;
+                            let expr5_0 = constructor_not_reg(ctx, pattern4_0, expr4_0)?;
+                            let expr6_0 = constructor_or_reg(ctx, pattern4_0, expr5_0, expr2_0)?;
+                            let expr7_0 = C::value_reg(ctx, expr6_0);
+                            return Some(expr7_0);
+                        }
+                    }
+                    &InstructionData::Unary {
+                        opcode: ref pattern6_0,
+                        arg: pattern6_1,
+                    } => {
+                        if let &Opcode::Bnot = &pattern6_0 {
+                            // Rule at src/isa/s390x/lower.isle line 625.
+                            let expr0_0 = C::put_in_reg(ctx, pattern6_1);
+                            let expr1_0 = constructor_not_reg(ctx, pattern4_0, expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
+        if let Some(()) = C::vxrs_ext2_enabled(ctx, pattern2_0) {
+            if pattern2_0 == F32 {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::Load {
+                    opcode: ref pattern6_0,
+                    arg: pattern6_1,
+                    flags: pattern6_2,
+                    offset: pattern6_3,
+                } = &pattern5_0
+                {
+                    if let &Opcode::Load = &pattern6_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern6_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1221.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern6_2, pattern6_1, pattern6_3)?;
+                            let expr1_0 = constructor_fpu_loadrev32(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+            }
+            if pattern2_0 == F64 {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::Load {
+                    opcode: ref pattern6_0,
+                    arg: pattern6_1,
+                    flags: pattern6_2,
+                    offset: pattern6_3,
+                } = &pattern5_0
+                {
+                    if let &Opcode::Load = &pattern6_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern6_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1236.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern6_2, pattern6_1, pattern6_3)?;
+                            let expr1_0 = constructor_fpu_loadrev64(ctx, &expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                    }
+                }
+            }
+        }
+        if let Some(()) = C::vxrs_ext2_disabled(ctx, pattern2_0) {
+            if pattern2_0 == F32 {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::Load {
+                    opcode: ref pattern6_0,
+                    arg: pattern6_1,
+                    flags: pattern6_2,
+                    offset: pattern6_3,
+                } = &pattern5_0
+                {
+                    if let &Opcode::Load = &pattern6_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern6_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1226.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern6_2, pattern6_1, pattern6_3)?;
+                            let expr1_0 = constructor_loadrev32(ctx, &expr0_0)?;
+                            let expr2_0: Type = I64;
+                            let expr3_0: u8 = 32;
+                            let expr4_0 = constructor_lshl_imm(ctx, expr2_0, expr1_0, expr3_0)?;
+                            let expr5_0 = constructor_mov_to_fpr(ctx, expr4_0)?;
+                            let expr6_0 = C::value_reg(ctx, expr5_0);
+                            return Some(expr6_0);
+                        }
+                    }
+                }
+            }
+            if pattern2_0 == F64 {
+                let pattern5_0 = C::inst_data(ctx, pattern0_0);
+                if let &InstructionData::Load {
+                    opcode: ref pattern6_0,
+                    arg: pattern6_1,
+                    flags: pattern6_2,
+                    offset: pattern6_3,
+                } = &pattern5_0
+                {
+                    if let &Opcode::Load = &pattern6_0 {
+                        if let Some(()) = C::littleendian(ctx, pattern6_2) {
+                            // Rule at src/isa/s390x/lower.isle line 1241.
+                            let expr0_0 =
+                                constructor_lower_address(ctx, pattern6_2, pattern6_1, pattern6_3)?;
+                            let expr1_0 = constructor_loadrev64(ctx, &expr0_0)?;
+                            let expr2_0 = constructor_mov_to_fpr(ctx, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                    }
+                }
+            }
+        }
+        if let Some(pattern3_0) = C::fits_in_16(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Unary {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+            } = &pattern4_0
+            {
+                if let &Opcode::Bint = &pattern5_0 {
+                    // Rule at src/isa/s390x/lower.isle line 791.
+                    let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                    let expr1_0: u16 = 1;
+                    let expr2_0: u8 = 0;
+                    let expr3_0 = C::uimm16shifted(ctx, expr1_0, expr2_0);
+                    let expr4_0 = constructor_and_uimm16shifted(ctx, pattern3_0, expr0_0, expr3_0)?;
+                    let expr5_0 = C::value_reg(ctx, expr4_0);
+                    return Some(expr5_0);
+                }
+            }
+        }
+        if let Some(pattern3_0) = C::fits_in_32(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Unary {
+                opcode: ref pattern5_0,
+                arg: pattern5_1,
+            } = &pattern4_0
+            {
+                if let &Opcode::Bint = &pattern5_0 {
+                    // Rule at src/isa/s390x/lower.isle line 795.
+                    let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                    let expr1_0: u32 = 1;
+                    let expr2_0: u8 = 0;
+                    let expr3_0 = C::uimm32shifted(ctx, expr1_0, expr2_0);
+                    let expr4_0 = constructor_and_uimm32shifted(ctx, pattern3_0, expr0_0, expr3_0)?;
+                    let expr5_0 = C::value_reg(ctx, expr4_0);
+                    return Some(expr5_0);
+                }
+            }
+        }
+        if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Binary {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Iadd => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i16_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 67.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_add_simm16(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::i32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 71.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_add_simm32(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::def_inst(ctx, pattern7_0) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Sextend = &pattern10_0 {
+                                        let pattern12_0 = C::value_type(ctx, pattern10_1);
+                                        if pattern12_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 61.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                            let expr1_0 = C::put_in_reg(ctx, pattern10_1);
+                                            let expr2_0 = constructor_add_reg_sext32(
+                                                ctx, pattern3_0, expr0_0, expr1_0,
+                                            )?;
+                                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Load {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                    flags: pattern10_2,
+                                    offset: pattern10_3,
+                                } = &pattern9_0
+                                {
+                                    match &pattern10_0 {
+                                        &Opcode::Sload16 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 89.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_sload16(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_add_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        &Opcode::Sload32 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 93.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_sload32(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_add_mem_sext32(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_0);
+                            if pattern8_0 == I16 {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 83.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_add_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 77.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_add_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::i16_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 65.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_add_simm16(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::i32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 69.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_add_simm32(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Sextend = &pattern10_0 {
+                                        let pattern12_0 = C::value_type(ctx, pattern10_1);
+                                        if pattern12_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 59.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                            let expr1_0 = C::put_in_reg(ctx, pattern10_1);
+                                            let expr2_0 = constructor_add_reg_sext32(
+                                                ctx, pattern3_0, expr0_0, expr1_0,
+                                            )?;
+                                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Load {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                    flags: pattern10_2,
+                                    offset: pattern10_3,
+                                } = &pattern9_0
+                                {
+                                    match &pattern10_0 {
+                                        &Opcode::Sload16 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 87.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_sload16(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_add_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        &Opcode::Sload32 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 91.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_sload32(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_add_mem_sext32(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_1);
+                            if pattern8_0 == I16 {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 81.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_add_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 75.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_add_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 55.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_add_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Isub => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i16_from_negated_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 108.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_add_simm16(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::i32_from_negated_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 110.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_add_simm32(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Sextend = &pattern10_0 {
+                                        let pattern12_0 = C::value_type(ctx, pattern10_1);
+                                        if pattern12_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 104.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                            let expr1_0 = C::put_in_reg(ctx, pattern10_1);
+                                            let expr2_0 = constructor_sub_reg_sext32(
+                                                ctx, pattern3_0, expr0_0, expr1_0,
+                                            )?;
+                                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Load {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                    flags: pattern10_2,
+                                    offset: pattern10_3,
+                                } = &pattern9_0
+                                {
+                                    match &pattern10_0 {
+                                        &Opcode::Sload16 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 122.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_sload16(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_sub_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        &Opcode::Sload32 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 124.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_sload32(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_sub_mem_sext32(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_1);
+                            if pattern8_0 == I16 {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 118.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_sub_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 114.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_sub_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 100.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_sub_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Imul => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i16_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 207.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_mul_simm16(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::i32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 211.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 =
+                                    constructor_mul_simm32(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::def_inst(ctx, pattern7_0) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Sextend = &pattern10_0 {
+                                        let pattern12_0 = C::value_type(ctx, pattern10_1);
+                                        if pattern12_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 201.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                            let expr1_0 = C::put_in_reg(ctx, pattern10_1);
+                                            let expr2_0 = constructor_mul_reg_sext32(
+                                                ctx, pattern3_0, expr0_0, expr1_0,
+                                            )?;
+                                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Load {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                    flags: pattern10_2,
+                                    offset: pattern10_3,
+                                } = &pattern9_0
+                                {
+                                    match &pattern10_0 {
+                                        &Opcode::Sload16 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 229.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_sload16(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_mul_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        &Opcode::Sload32 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 233.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_sload32(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_mul_mem_sext32(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_0);
+                            if pattern8_0 == I16 {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 223.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_mul_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 217.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_mul_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::i16_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 205.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_mul_simm16(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::i32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 209.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 =
+                                    constructor_mul_simm32(ctx, pattern3_0, expr0_0, pattern8_0)?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Sextend = &pattern10_0 {
+                                        let pattern12_0 = C::value_type(ctx, pattern10_1);
+                                        if pattern12_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 199.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                            let expr1_0 = C::put_in_reg(ctx, pattern10_1);
+                                            let expr2_0 = constructor_mul_reg_sext32(
+                                                ctx, pattern3_0, expr0_0, expr1_0,
+                                            )?;
+                                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Load {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                    flags: pattern10_2,
+                                    offset: pattern10_3,
+                                } = &pattern9_0
+                                {
+                                    match &pattern10_0 {
+                                        &Opcode::Sload16 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 227.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_sload16(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_mul_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        &Opcode::Sload32 => {
+                                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 231.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_sload32(ctx, pattern8_0)?;
+                                                let expr2_0 = constructor_mul_mem_sext32(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_1);
+                            if pattern8_0 == I16 {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 221.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_mul_mem_sext16(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 215.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_mul_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 195.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_mul_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Udiv => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 298.
+                            let expr0_0 = constructor_zero_divisor_check_needed(ctx, pattern7_1)?;
+                            let expr1_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr2_0: u64 = 0;
+                            let expr3_0 = constructor_uninitialized_regpair(ctx)?;
+                            let expr4_0 =
+                                constructor_imm_regpair_hi(ctx, expr1_0, expr2_0, &expr3_0)?;
+                            let expr5_0 =
+                                constructor_put_in_regpair_lo_zext32(ctx, pattern7_0, &expr4_0)?;
+                            let expr6_0 = constructor_put_in_reg_zext32(ctx, pattern7_1)?;
+                            let expr7_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr8_0 = constructor_maybe_trap_if_zero_divisor(
+                                ctx, expr0_0, expr7_0, expr6_0,
+                            )?;
+                            let expr9_0 = constructor_udivmod(ctx, expr7_0, &expr5_0, expr6_0)?;
+                            let expr10_0 = constructor_regpair_lo(ctx, &expr9_0)?;
+                            let expr11_0 = constructor_copy_reg(ctx, pattern3_0, expr10_0)?;
+                            let expr12_0 = C::value_reg(ctx, expr11_0);
+                            return Some(expr12_0);
+                        }
+                        &Opcode::Sdiv => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 370.
+                            let expr0_0 = constructor_zero_divisor_check_needed(ctx, pattern7_1)?;
+                            let expr1_0 = constructor_div_overflow_check_needed(ctx, pattern7_1)?;
+                            let expr2_0 = constructor_uninitialized_regpair(ctx)?;
+                            let expr3_0 =
+                                constructor_put_in_regpair_lo_sext64(ctx, pattern7_0, &expr2_0)?;
+                            let expr4_0 = constructor_put_in_reg_sext32(ctx, pattern7_1)?;
+                            let expr5_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr6_0 = constructor_maybe_trap_if_zero_divisor(
+                                ctx, expr0_0, expr5_0, expr4_0,
+                            )?;
+                            let expr7_0 = constructor_maybe_trap_if_sdiv_overflow(
+                                ctx, expr1_0, expr5_0, pattern3_0, &expr3_0, expr4_0,
+                            )?;
+                            let expr8_0 = constructor_sdivmod(ctx, expr5_0, &expr3_0, expr4_0)?;
+                            let expr9_0 = constructor_regpair_lo(ctx, &expr8_0)?;
+                            let expr10_0 = constructor_copy_reg(ctx, pattern3_0, expr9_0)?;
+                            let expr11_0 = C::value_reg(ctx, expr10_0);
+                            return Some(expr11_0);
+                        }
+                        &Opcode::Urem => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 321.
+                            let expr0_0 = constructor_zero_divisor_check_needed(ctx, pattern7_1)?;
+                            let expr1_0: u64 = 0;
+                            let expr2_0 = constructor_uninitialized_regpair(ctx)?;
+                            let expr3_0 =
+                                constructor_imm_regpair_hi(ctx, pattern3_0, expr1_0, &expr2_0)?;
+                            let expr4_0 =
+                                constructor_put_in_regpair_lo_zext32(ctx, pattern7_0, &expr3_0)?;
+                            let expr5_0 = constructor_put_in_reg_zext32(ctx, pattern7_1)?;
+                            let expr6_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr7_0 = constructor_maybe_trap_if_zero_divisor(
+                                ctx, expr0_0, expr6_0, expr5_0,
+                            )?;
+                            let expr8_0 = constructor_udivmod(ctx, expr6_0, &expr4_0, expr5_0)?;
+                            let expr9_0 = constructor_regpair_hi(ctx, &expr8_0)?;
+                            let expr10_0 = constructor_copy_reg(ctx, pattern3_0, expr9_0)?;
+                            let expr11_0 = C::value_reg(ctx, expr10_0);
+                            return Some(expr11_0);
+                        }
+                        &Opcode::Srem => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 393.
+                            let expr0_0 = constructor_zero_divisor_check_needed(ctx, pattern7_1)?;
+                            let expr1_0 = constructor_div_overflow_check_needed(ctx, pattern7_1)?;
+                            let expr2_0 = constructor_uninitialized_regpair(ctx)?;
+                            let expr3_0 =
+                                constructor_put_in_regpair_lo_sext64(ctx, pattern7_0, &expr2_0)?;
+                            let expr4_0 = constructor_put_in_reg_sext32(ctx, pattern7_1)?;
+                            let expr5_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr6_0 = constructor_maybe_trap_if_zero_divisor(
+                                ctx, expr0_0, expr5_0, expr4_0,
+                            )?;
+                            let expr7_0 = constructor_maybe_avoid_srem_overflow(
+                                ctx, expr1_0, expr5_0, &expr3_0, expr4_0,
+                            )?;
+                            let expr8_0 = constructor_sdivmod(ctx, expr5_0, &expr7_0, expr4_0)?;
+                            let expr9_0 = constructor_regpair_hi(ctx, &expr8_0)?;
+                            let expr10_0 = constructor_copy_reg(ctx, pattern3_0, expr9_0)?;
+                            let expr11_0 = C::value_reg(ctx, expr10_0);
+                            return Some(expr11_0);
+                        }
+                        &Opcode::IaddIfcout => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::u32_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 165.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = constructor_add_logical_zimm32(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = constructor_value_regs_ifcout(ctx, expr1_0)?;
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::def_inst(ctx, pattern7_0) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Uextend = &pattern10_0 {
+                                        let pattern12_0 = C::value_type(ctx, pattern10_1);
+                                        if pattern12_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 159.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                            let expr1_0 = C::put_in_reg(ctx, pattern10_1);
+                                            let expr2_0 = constructor_add_logical_reg_zext32(
+                                                ctx, pattern3_0, expr0_0, expr1_0,
+                                            )?;
+                                            let expr3_0 =
+                                                constructor_value_regs_ifcout(ctx, expr2_0)?;
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Load {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                    flags: pattern10_2,
+                                    offset: pattern10_3,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Uload32 = &pattern10_0 {
+                                        if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                            // Rule at src/isa/s390x/lower.isle line 177.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                            let expr1_0 =
+                                                constructor_sink_uload32(ctx, pattern8_0)?;
+                                            let expr2_0 = constructor_add_logical_mem_zext32(
+                                                ctx, pattern3_0, expr0_0, &expr1_0,
+                                            )?;
+                                            let expr3_0 =
+                                                constructor_value_regs_ifcout(ctx, expr2_0)?;
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_0);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 171.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_add_logical_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 =
+                                                    constructor_value_regs_ifcout(ctx, expr2_0)?;
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::u32_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 163.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = constructor_add_logical_zimm32(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = constructor_value_regs_ifcout(ctx, expr1_0)?;
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Uextend = &pattern10_0 {
+                                        let pattern12_0 = C::value_type(ctx, pattern10_1);
+                                        if pattern12_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 157.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                            let expr1_0 = C::put_in_reg(ctx, pattern10_1);
+                                            let expr2_0 = constructor_add_logical_reg_zext32(
+                                                ctx, pattern3_0, expr0_0, expr1_0,
+                                            )?;
+                                            let expr3_0 =
+                                                constructor_value_regs_ifcout(ctx, expr2_0)?;
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                                if let &InstructionData::Load {
+                                    opcode: ref pattern10_0,
+                                    arg: pattern10_1,
+                                    flags: pattern10_2,
+                                    offset: pattern10_3,
+                                } = &pattern9_0
+                                {
+                                    if let &Opcode::Uload32 = &pattern10_0 {
+                                        if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                            // Rule at src/isa/s390x/lower.isle line 175.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                            let expr1_0 =
+                                                constructor_sink_uload32(ctx, pattern8_0)?;
+                                            let expr2_0 = constructor_add_logical_mem_zext32(
+                                                ctx, pattern3_0, expr0_0, &expr1_0,
+                                            )?;
+                                            let expr3_0 =
+                                                constructor_value_regs_ifcout(ctx, expr2_0)?;
+                                            return Some(expr3_0);
+                                        }
+                                    }
+                                }
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_1);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 169.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_add_logical_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 =
+                                                    constructor_value_regs_ifcout(ctx, expr2_0)?;
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 153.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 =
+                                constructor_add_logical_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = constructor_value_regs_ifcout(ctx, expr2_0)?;
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Band => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            let pattern8_0 = C::value_type(ctx, pattern7_0);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 648.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_and_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) =
+                                C::uimm32shifted_from_inverted_value(ctx, pattern7_0)
+                            {
+                                // Rule at src/isa/s390x/lower.isle line 642.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = constructor_and_uimm32shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) =
+                                C::uimm16shifted_from_inverted_value(ctx, pattern7_0)
+                            {
+                                // Rule at src/isa/s390x/lower.isle line 638.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = constructor_and_uimm16shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_1);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 646.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_and_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) =
+                                C::uimm32shifted_from_inverted_value(ctx, pattern7_1)
+                            {
+                                // Rule at src/isa/s390x/lower.isle line 640.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = constructor_and_uimm32shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) =
+                                C::uimm16shifted_from_inverted_value(ctx, pattern7_1)
+                            {
+                                // Rule at src/isa/s390x/lower.isle line 636.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = constructor_and_uimm16shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 632.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_and_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Bor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            let pattern8_0 = C::value_type(ctx, pattern7_0);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 671.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_or_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::uimm32shifted_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 665.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = constructor_or_uimm32shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::uimm16shifted_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 661.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = constructor_or_uimm16shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_1);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 669.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_or_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::uimm32shifted_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 663.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = constructor_or_uimm32shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            if let Some(pattern8_0) = C::uimm16shifted_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 659.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = constructor_or_uimm16shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 655.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_or_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Bxor => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            let pattern8_0 = C::value_type(ctx, pattern7_0);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_0) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 690.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_xor_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::uimm32shifted_from_value(ctx, pattern7_0) {
+                                // Rule at src/isa/s390x/lower.isle line 684.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                let expr1_0 = constructor_xor_uimm32shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            let pattern8_0 = C::value_type(ctx, pattern7_1);
+                            if let Some(pattern9_0) = C::ty_32_or_64(ctx, pattern8_0) {
+                                if let Some(pattern10_0) = C::sinkable_inst(ctx, pattern7_1) {
+                                    let pattern11_0 = C::inst_data(ctx, pattern10_0);
+                                    if let &InstructionData::Load {
+                                        opcode: ref pattern12_0,
+                                        arg: pattern12_1,
+                                        flags: pattern12_2,
+                                        offset: pattern12_3,
+                                    } = &pattern11_0
+                                    {
+                                        if let &Opcode::Load = &pattern12_0 {
+                                            if let Some(()) = C::bigendian(ctx, pattern12_2) {
+                                                // Rule at src/isa/s390x/lower.isle line 688.
+                                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                                let expr1_0 =
+                                                    constructor_sink_load(ctx, pattern10_0)?;
+                                                let expr2_0 = constructor_xor_mem(
+                                                    ctx, pattern3_0, expr0_0, &expr1_0,
+                                                )?;
+                                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                                return Some(expr3_0);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if let Some(pattern8_0) = C::uimm32shifted_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 682.
+                                let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr1_0 = constructor_xor_uimm32shifted(
+                                    ctx, pattern3_0, expr0_0, pattern8_0,
+                                )?;
+                                let expr2_0 = C::value_reg(ctx, expr1_0);
+                                return Some(expr2_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 678.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_xor_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Ishl => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i64_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 477.
+                                let expr0_0 = C::mask_amt_imm(ctx, pattern3_0, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr2_0 =
+                                    constructor_lshl_imm(ctx, pattern3_0, expr1_0, expr0_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 472.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr1_0 = constructor_mask_amt_reg(ctx, pattern3_0, expr0_0)?;
+                            let expr2_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr3_0 = constructor_lshl_reg(ctx, pattern3_0, expr2_0, expr1_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        &Opcode::Ushr => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i64_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 493.
+                                let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
+                                let expr1_0 = C::mask_amt_imm(ctx, pattern3_0, pattern8_0);
+                                let expr2_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                                let expr3_0 = constructor_lshr_imm(ctx, expr2_0, expr0_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 486.
+                            let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_mask_amt_reg(ctx, pattern3_0, expr1_0)?;
+                            let expr3_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr4_0 = constructor_lshr_reg(ctx, expr3_0, expr0_0, expr2_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                        &Opcode::Sshr => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i64_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 510.
+                                let expr0_0 = constructor_put_in_reg_sext32(ctx, pattern7_0)?;
+                                let expr1_0 = C::mask_amt_imm(ctx, pattern3_0, pattern8_0);
+                                let expr2_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                                let expr3_0 = constructor_ashr_imm(ctx, expr2_0, expr0_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 503.
+                            let expr0_0 = constructor_put_in_reg_sext32(ctx, pattern7_0)?;
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_mask_amt_reg(ctx, pattern3_0, expr1_0)?;
+                            let expr3_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr4_0 = constructor_ashr_reg(ctx, expr3_0, expr0_0, expr2_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Ineg => {
+                            if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) {
+                                let pattern8_0 = C::inst_data(ctx, pattern7_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern9_0,
+                                    arg: pattern9_1,
+                                } = &pattern8_0
+                                {
+                                    if let &Opcode::Sextend = &pattern9_0 {
+                                        let pattern11_0 = C::value_type(ctx, pattern9_1);
+                                        if pattern11_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 188.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern9_1);
+                                            let expr1_0 = constructor_neg_reg_sext32(
+                                                ctx, pattern3_0, expr0_0,
+                                            )?;
+                                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                                            return Some(expr2_0);
+                                        }
+                                    }
+                                }
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 184.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0 = constructor_neg_reg(ctx, pattern3_0, expr0_0)?;
+                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                            return Some(expr2_0);
+                        }
+                        &Opcode::Iabs => {
+                            if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) {
+                                let pattern8_0 = C::inst_data(ctx, pattern7_0);
+                                if let &InstructionData::Unary {
+                                    opcode: ref pattern9_0,
+                                    arg: pattern9_1,
+                                } = &pattern8_0
+                                {
+                                    if let &Opcode::Sextend = &pattern9_0 {
+                                        let pattern11_0 = C::value_type(ctx, pattern9_1);
+                                        if pattern11_0 == I32 {
+                                            // Rule at src/isa/s390x/lower.isle line 136.
+                                            let expr0_0 = C::put_in_reg(ctx, pattern9_1);
+                                            let expr1_0 = constructor_abs_reg_sext32(
+                                                ctx, pattern3_0, expr0_0,
+                                            )?;
+                                            let expr2_0 = C::value_reg(ctx, expr1_0);
+                                            return Some(expr2_0);
+                                        }
+                                    }
+                                }
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 132.
+                            let expr0_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                            let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern5_1)?;
+                            let expr2_0 = constructor_abs_reg(ctx, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Clz => {
+                            // Rule at src/isa/s390x/lower.isle line 816.
+                            let expr0_0 = constructor_put_in_reg_zext64(ctx, pattern5_1)?;
+                            let expr1_0: i16 = 64;
+                            let expr2_0 = constructor_clz_reg(ctx, expr1_0, expr0_0)?;
+                            let expr3_0 = constructor_regpair_hi(ctx, &expr2_0)?;
+                            let expr4_0 = constructor_clz_offset(ctx, pattern3_0, expr3_0)?;
+                            let expr5_0 = C::value_reg(ctx, expr4_0);
+                            return Some(expr5_0);
+                        }
+                        &Opcode::Cls => {
+                            // Rule at src/isa/s390x/lower.isle line 831.
+                            let expr0_0 = constructor_put_in_reg_sext64(ctx, pattern5_1)?;
+                            let expr1_0: Type = I64;
+                            let expr2_0: u8 = 63;
+                            let expr3_0 = constructor_ashr_imm(ctx, expr1_0, expr0_0, expr2_0)?;
+                            let expr4_0: Type = I64;
+                            let expr5_0 = constructor_xor_reg(ctx, expr4_0, expr0_0, expr3_0)?;
+                            let expr6_0: i16 = 64;
+                            let expr7_0 = constructor_clz_reg(ctx, expr6_0, expr5_0)?;
+                            let expr8_0 = constructor_regpair_hi(ctx, &expr7_0)?;
+                            let expr9_0 = constructor_clz_offset(ctx, pattern3_0, expr8_0)?;
+                            let expr10_0 = C::value_reg(ctx, expr9_0);
+                            return Some(expr10_0);
+                        }
+                        &Opcode::Bint => {
+                            // Rule at src/isa/s390x/lower.isle line 799.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0: u64 = 1;
+                            let expr2_0 = constructor_imm(ctx, pattern3_0, expr1_0)?;
+                            let expr3_0 = constructor_and_reg(ctx, pattern3_0, expr0_0, expr2_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        _ => {}
+                    }
+                }
+                _ => {}
+            }
+        }
+        if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Binary {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Rotl => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i64_from_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 523.
+                                let expr0_0 = C::mask_amt_imm(ctx, pattern3_0, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr2_0 =
+                                    constructor_rot_imm(ctx, pattern3_0, expr1_0, expr0_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 519.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr2_0 = constructor_rot_reg(ctx, pattern3_0, expr0_0, expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Rotr => {
+                            let (pattern7_0, pattern7_1) =
+                                C::unpack_value_array_2(ctx, &pattern5_1);
+                            if let Some(pattern8_0) = C::i64_from_negated_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 561.
+                                let expr0_0 = C::mask_amt_imm(ctx, pattern3_0, pattern8_0);
+                                let expr1_0 = C::put_in_reg(ctx, pattern7_0);
+                                let expr2_0 =
+                                    constructor_rot_imm(ctx, pattern3_0, expr1_0, expr0_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                            // Rule at src/isa/s390x/lower.isle line 555.
+                            let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                            let expr1_0 = constructor_neg_reg(ctx, pattern3_0, expr0_0)?;
+                            let expr2_0 = C::put_in_reg(ctx, pattern7_0);
+                            let expr3_0 = constructor_rot_reg(ctx, pattern3_0, expr2_0, expr1_0)?;
+                            let expr4_0 = C::value_reg(ctx, expr3_0);
+                            return Some(expr4_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::AtomicRmw {
+                    opcode: ref pattern5_0,
+                    args: ref pattern5_1,
+                    flags: pattern5_2,
+                    op: ref pattern5_3,
+                } => {
+                    if let &Opcode::AtomicRmw = &pattern5_0 {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                            match &pattern5_3 {
+                                &AtomicRmwOp::Add => {
+                                    // Rule at src/isa/s390x/lower.isle line 1514.
+                                    let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                    let expr1_0 = C::zero_offset(ctx);
+                                    let expr2_0 = constructor_lower_address(
+                                        ctx, pattern5_2, pattern7_0, expr1_0,
+                                    )?;
+                                    let expr3_0 = constructor_atomic_rmw_add(
+                                        ctx, pattern3_0, expr0_0, &expr2_0,
+                                    )?;
+                                    let expr4_0 = C::value_reg(ctx, expr3_0);
+                                    return Some(expr4_0);
+                                }
+                                &AtomicRmwOp::And => {
+                                    // Rule at src/isa/s390x/lower.isle line 1496.
+                                    let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                    let expr1_0 = C::zero_offset(ctx);
+                                    let expr2_0 = constructor_lower_address(
+                                        ctx, pattern5_2, pattern7_0, expr1_0,
+                                    )?;
+                                    let expr3_0 = constructor_atomic_rmw_and(
+                                        ctx, pattern3_0, expr0_0, &expr2_0,
+                                    )?;
+                                    let expr4_0 = C::value_reg(ctx, expr3_0);
+                                    return Some(expr4_0);
+                                }
+                                &AtomicRmwOp::Or => {
+                                    // Rule at src/isa/s390x/lower.isle line 1502.
+                                    let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                    let expr1_0 = C::zero_offset(ctx);
+                                    let expr2_0 = constructor_lower_address(
+                                        ctx, pattern5_2, pattern7_0, expr1_0,
+                                    )?;
+                                    let expr3_0 = constructor_atomic_rmw_or(
+                                        ctx, pattern3_0, expr0_0, &expr2_0,
+                                    )?;
+                                    let expr4_0 = C::value_reg(ctx, expr3_0);
+                                    return Some(expr4_0);
+                                }
+                                &AtomicRmwOp::Sub => {
+                                    // Rule at src/isa/s390x/lower.isle line 1520.
+                                    let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                    let expr1_0 = constructor_neg_reg(ctx, pattern3_0, expr0_0)?;
+                                    let expr2_0 = C::zero_offset(ctx);
+                                    let expr3_0 = constructor_lower_address(
+                                        ctx, pattern5_2, pattern7_0, expr2_0,
+                                    )?;
+                                    let expr4_0 = constructor_atomic_rmw_add(
+                                        ctx, pattern3_0, expr1_0, &expr3_0,
+                                    )?;
+                                    let expr5_0 = C::value_reg(ctx, expr4_0);
+                                    return Some(expr5_0);
+                                }
+                                &AtomicRmwOp::Xor => {
+                                    // Rule at src/isa/s390x/lower.isle line 1508.
+                                    let expr0_0 = C::put_in_reg(ctx, pattern7_1);
+                                    let expr1_0 = C::zero_offset(ctx);
+                                    let expr2_0 = constructor_lower_address(
+                                        ctx, pattern5_2, pattern7_0, expr1_0,
+                                    )?;
+                                    let expr3_0 = constructor_atomic_rmw_xor(
+                                        ctx, pattern3_0, expr0_0, &expr2_0,
+                                    )?;
+                                    let expr4_0 = C::value_reg(ctx, expr3_0);
+                                    return Some(expr4_0);
+                                }
+                                _ => {}
+                            }
+                        }
+                    }
+                }
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::FcvtToUint => {
+                            let pattern7_0 = C::value_type(ctx, pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 1052.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0 = constructor_fcmp_reg(ctx, pattern7_0, expr0_0, expr0_0)?;
+                            let expr2_0 = FloatCC::Unordered;
+                            let expr3_0 = C::floatcc_as_cond(ctx, &expr2_0);
+                            let expr4_0 = C::trap_code_bad_conversion_to_integer(ctx);
+                            let expr5_0 = constructor_trap_if(ctx, &expr1_0, &expr3_0, &expr4_0)?;
+                            let expr6_0 = constructor_fcvt_to_uint_reg_with_flags(
+                                ctx, pattern3_0, pattern7_0, expr0_0,
+                            )?;
+                            let expr7_0 = FloatCC::Unordered;
+                            let expr8_0 = C::floatcc_as_cond(ctx, &expr7_0);
+                            let expr9_0 = C::trap_code_integer_overflow(ctx);
+                            let expr10_0 = constructor_trap_if(ctx, &expr6_0, &expr8_0, &expr9_0)?;
+                            let expr11_0 = C::value_reg(ctx, expr10_0);
+                            return Some(expr11_0);
+                        }
+                        &Opcode::FcvtToUintSat => {
+                            let pattern7_0 = C::value_type(ctx, pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 1093.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0 =
+                                constructor_fcvt_to_uint_reg(ctx, pattern3_0, pattern7_0, expr0_0)?;
+                            let expr2_0 = constructor_fcmp_reg(ctx, pattern7_0, expr0_0, expr0_0)?;
+                            let expr3_0 = FloatCC::Unordered;
+                            let expr4_0 = C::floatcc_as_cond(ctx, &expr3_0);
+                            let expr5_0: i16 = 0;
+                            let expr6_0 =
+                                constructor_cmov_imm(ctx, pattern3_0, &expr4_0, expr5_0, expr1_0)?;
+                            let expr7_0 = constructor_with_flags_1(ctx, &expr2_0, &expr6_0)?;
+                            let expr8_0 = C::value_reg(ctx, expr7_0);
+                            return Some(expr8_0);
+                        }
+                        &Opcode::FcvtToSint => {
+                            let pattern7_0 = C::value_type(ctx, pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 1073.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0 = constructor_fcmp_reg(ctx, pattern7_0, expr0_0, expr0_0)?;
+                            let expr2_0 = FloatCC::Unordered;
+                            let expr3_0 = C::floatcc_as_cond(ctx, &expr2_0);
+                            let expr4_0 = C::trap_code_bad_conversion_to_integer(ctx);
+                            let expr5_0 = constructor_trap_if(ctx, &expr1_0, &expr3_0, &expr4_0)?;
+                            let expr6_0 = constructor_fcvt_to_sint_reg_with_flags(
+                                ctx, pattern3_0, pattern7_0, expr0_0,
+                            )?;
+                            let expr7_0 = FloatCC::Unordered;
+                            let expr8_0 = C::floatcc_as_cond(ctx, &expr7_0);
+                            let expr9_0 = C::trap_code_integer_overflow(ctx);
+                            let expr10_0 = constructor_trap_if(ctx, &expr6_0, &expr8_0, &expr9_0)?;
+                            let expr11_0 = C::value_reg(ctx, expr10_0);
+                            return Some(expr11_0);
+                        }
+                        &Opcode::FcvtToSintSat => {
+                            let pattern7_0 = C::value_type(ctx, pattern5_1);
+                            // Rule at src/isa/s390x/lower.isle line 1110.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0 =
+                                constructor_fcvt_to_sint_reg(ctx, pattern3_0, pattern7_0, expr0_0)?;
+                            let expr2_0 = constructor_fcmp_reg(ctx, pattern7_0, expr0_0, expr0_0)?;
+                            let expr3_0 = FloatCC::Unordered;
+                            let expr4_0 = C::floatcc_as_cond(ctx, &expr3_0);
+                            let expr5_0: i16 = 0;
+                            let expr6_0 =
+                                constructor_cmov_imm(ctx, pattern3_0, &expr4_0, expr5_0, expr1_0)?;
+                            let expr7_0 = constructor_with_flags_1(ctx, &expr2_0, &expr6_0)?;
+                            let expr8_0 = C::value_reg(ctx, expr7_0);
+                            return Some(expr8_0);
+                        }
+                        _ => {}
+                    }
+                }
+                _ => {}
+            }
+        }
+        if let Some(pattern3_0) = C::ty_8_or_16(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            if let &InstructionData::Binary {
+                opcode: ref pattern5_0,
+                args: ref pattern5_1,
+            } = &pattern4_0
+            {
+                match &pattern5_0 {
+                    &Opcode::Umulhi => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/s390x/lower.isle line 240.
+                        let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
+                        let expr1_0 = constructor_put_in_reg_zext32(ctx, pattern7_1)?;
+                        let expr2_0: Type = I32;
+                        let expr3_0 = constructor_mul_reg(ctx, expr2_0, expr0_0, expr1_0)?;
+                        let expr4_0: Type = I32;
+                        let expr5_0 = C::ty_bits(ctx, pattern3_0);
+                        let expr6_0 = constructor_lshr_imm(ctx, expr4_0, expr3_0, expr5_0)?;
+                        let expr7_0 = C::value_reg(ctx, expr6_0);
+                        return Some(expr7_0);
+                    }
+                    &Opcode::Smulhi => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        // Rule at src/isa/s390x/lower.isle line 262.
+                        let expr0_0 = constructor_put_in_reg_sext32(ctx, pattern7_0)?;
+                        let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern7_1)?;
+                        let expr2_0: Type = I32;
+                        let expr3_0 = constructor_mul_reg(ctx, expr2_0, expr0_0, expr1_0)?;
+                        let expr4_0: Type = I32;
+                        let expr5_0 = C::ty_bits(ctx, pattern3_0);
+                        let expr6_0 = constructor_ashr_imm(ctx, expr4_0, expr3_0, expr5_0)?;
+                        let expr7_0 = C::value_reg(ctx, expr6_0);
+                        return Some(expr7_0);
+                    }
+                    &Opcode::Rotl => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        if let Some(pattern8_0) = C::i64_from_value(ctx, pattern7_1) {
+                            if let Some(pattern9_0) = C::i64_from_negated_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 541.
+                                let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
+                                let expr1_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                                let expr2_0 = C::mask_amt_imm(ctx, pattern3_0, pattern8_0);
+                                let expr3_0 = C::mask_amt_imm(ctx, pattern3_0, pattern9_0);
+                                let expr4_0 = constructor_lshl_imm(ctx, expr1_0, expr0_0, expr2_0)?;
+                                let expr5_0 = constructor_lshr_imm(ctx, expr1_0, expr0_0, expr3_0)?;
+                                let expr6_0 =
+                                    constructor_or_reg(ctx, pattern3_0, expr4_0, expr5_0)?;
+                                let expr7_0 = C::value_reg(ctx, expr6_0);
+                                return Some(expr7_0);
+                            }
+                        }
+                        // Rule at src/isa/s390x/lower.isle line 529.
+                        let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
+                        let expr1_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                        let expr2_0 = C::put_in_reg(ctx, pattern7_1);
+                        let expr3_0 = constructor_neg_reg(ctx, pattern3_0, expr2_0)?;
+                        let expr4_0 = constructor_mask_amt_reg(ctx, pattern3_0, expr2_0)?;
+                        let expr5_0 = constructor_mask_amt_reg(ctx, pattern3_0, expr3_0)?;
+                        let expr6_0 = constructor_lshl_reg(ctx, expr1_0, expr0_0, expr4_0)?;
+                        let expr7_0 = constructor_lshr_reg(ctx, expr1_0, expr0_0, expr5_0)?;
+                        let expr8_0 = constructor_or_reg(ctx, pattern3_0, expr6_0, expr7_0)?;
+                        let expr9_0 = C::value_reg(ctx, expr8_0);
+                        return Some(expr9_0);
+                    }
+                    &Opcode::Rotr => {
+                        let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1);
+                        if let Some(pattern8_0) = C::i64_from_value(ctx, pattern7_1) {
+                            if let Some(pattern9_0) = C::i64_from_negated_value(ctx, pattern7_1) {
+                                // Rule at src/isa/s390x/lower.isle line 579.
+                                let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
+                                let expr1_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                                let expr2_0 = C::mask_amt_imm(ctx, pattern3_0, pattern8_0);
+                                let expr3_0 = C::mask_amt_imm(ctx, pattern3_0, pattern9_0);
+                                let expr4_0 = constructor_lshl_imm(ctx, expr1_0, expr0_0, expr3_0)?;
+                                let expr5_0 = constructor_lshr_imm(ctx, expr1_0, expr0_0, expr2_0)?;
+                                let expr6_0 =
+                                    constructor_or_reg(ctx, pattern3_0, expr4_0, expr5_0)?;
+                                let expr7_0 = C::value_reg(ctx, expr6_0);
+                                return Some(expr7_0);
+                            }
+                        }
+                        // Rule at src/isa/s390x/lower.isle line 567.
+                        let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern7_0)?;
+                        let expr1_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+                        let expr2_0 = C::put_in_reg(ctx, pattern7_1);
+                        let expr3_0 = constructor_neg_reg(ctx, pattern3_0, expr2_0)?;
+                        let expr4_0 = constructor_mask_amt_reg(ctx, pattern3_0, expr2_0)?;
+                        let expr5_0 = constructor_mask_amt_reg(ctx, pattern3_0, expr3_0)?;
+                        let expr6_0 = constructor_lshl_reg(ctx, expr1_0, expr0_0, expr5_0)?;
+                        let expr7_0 = constructor_lshr_reg(ctx, expr1_0, expr0_0, expr4_0)?;
+                        let expr8_0 = constructor_or_reg(ctx, pattern3_0, expr6_0, expr7_0)?;
+                        let expr9_0 = C::value_reg(ctx, expr8_0);
+                        return Some(expr9_0);
+                    }
+                    _ => {}
+                }
+            }
+        }
+        if let Some(pattern3_0) = C::gpr32_ty(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Ctz => {
+                            // Rule at src/isa/s390x/lower.isle line 854.
+                            let expr0_0: Type = I64;
+                            let expr1_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr2_0 = constructor_ctz_guardbit(ctx, pattern3_0)?;
+                            let expr3_0 =
+                                constructor_or_uimm16shifted(ctx, expr0_0, expr1_0, expr2_0)?;
+                            let expr4_0: Type = I64;
+                            let expr5_0: Type = I64;
+                            let expr6_0 = constructor_neg_reg(ctx, expr5_0, expr3_0)?;
+                            let expr7_0 = constructor_and_reg(ctx, expr4_0, expr3_0, expr6_0)?;
+                            let expr8_0: i16 = 64;
+                            let expr9_0 = constructor_clz_reg(ctx, expr8_0, expr7_0)?;
+                            let expr10_0: u64 = 63;
+                            let expr11_0 = constructor_imm(ctx, pattern3_0, expr10_0)?;
+                            let expr12_0 = constructor_regpair_hi(ctx, &expr9_0)?;
+                            let expr13_0 =
+                                constructor_sub_reg(ctx, pattern3_0, expr11_0, expr12_0)?;
+                            let expr14_0 = C::value_reg(ctx, expr13_0);
+                            return Some(expr14_0);
+                        }
+                        &Opcode::Uextend => {
+                            // Rule at src/isa/s390x/lower.isle line 598.
+                            let expr0_0 = constructor_put_in_reg_zext32(ctx, pattern5_1)?;
+                            let expr1_0 = C::value_reg(ctx, expr0_0);
+                            return Some(expr1_0);
+                        }
+                        &Opcode::Sextend => {
+                            // Rule at src/isa/s390x/lower.isle line 609.
+                            let expr0_0 = constructor_put_in_reg_sext32(ctx, pattern5_1)?;
+                            let expr1_0 = C::value_reg(ctx, expr0_0);
+                            return Some(expr1_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Uload8 => {
+                            // Rule at src/isa/s390x/lower.isle line 1250.
+                            let expr0_0: Type = I8;
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr2_0 = constructor_zext32_mem(ctx, expr0_0, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Sload8 => {
+                            // Rule at src/isa/s390x/lower.isle line 1261.
+                            let expr0_0: Type = I8;
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr2_0 = constructor_sext32_mem(ctx, expr0_0, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Uload16 => {
+                            if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1277.
+                                let expr0_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr1_0 = constructor_loadrev16(ctx, &expr0_0)?;
+                                let expr2_0: Type = I16;
+                                let expr3_0 = constructor_zext32_reg(ctx, expr2_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1272.
+                                let expr0_0: Type = I16;
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr2_0 = constructor_zext32_mem(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                        }
+                        &Opcode::Sload16 => {
+                            if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1302.
+                                let expr0_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr1_0 = constructor_loadrev16(ctx, &expr0_0)?;
+                                let expr2_0: Type = I16;
+                                let expr3_0 = constructor_sext32_reg(ctx, expr2_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1297.
+                                let expr0_0: Type = I16;
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr2_0 = constructor_sext32_mem(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+                _ => {}
+            }
+        }
+        if let Some(pattern3_0) = C::gpr64_ty(ctx, pattern2_0) {
+            let pattern4_0 = C::inst_data(ctx, pattern0_0);
+            match &pattern4_0 {
+                &InstructionData::Unary {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Ctz => {
+                            // Rule at src/isa/s390x/lower.isle line 869.
+                            let expr0_0 = C::put_in_reg(ctx, pattern5_1);
+                            let expr1_0: Type = I64;
+                            let expr2_0: Type = I64;
+                            let expr3_0 = constructor_neg_reg(ctx, expr2_0, expr0_0)?;
+                            let expr4_0 = constructor_and_reg(ctx, expr1_0, expr0_0, expr3_0)?;
+                            let expr5_0: i16 = -1;
+                            let expr6_0 = constructor_clz_reg(ctx, expr5_0, expr4_0)?;
+                            let expr7_0: Type = I64;
+                            let expr8_0: Type = I64;
+                            let expr9_0: u64 = 63;
+                            let expr10_0 = constructor_imm(ctx, expr8_0, expr9_0)?;
+                            let expr11_0 = constructor_regpair_hi(ctx, &expr6_0)?;
+                            let expr12_0 = constructor_sub_reg(ctx, expr7_0, expr10_0, expr11_0)?;
+                            let expr13_0 = C::value_reg(ctx, expr12_0);
+                            return Some(expr13_0);
+                        }
+                        &Opcode::Uextend => {
+                            // Rule at src/isa/s390x/lower.isle line 602.
+                            let expr0_0 = constructor_put_in_reg_zext64(ctx, pattern5_1)?;
+                            let expr1_0 = C::value_reg(ctx, expr0_0);
+                            return Some(expr1_0);
+                        }
+                        &Opcode::Sextend => {
+                            // Rule at src/isa/s390x/lower.isle line 613.
+                            let expr0_0 = constructor_put_in_reg_sext64(ctx, pattern5_1)?;
+                            let expr1_0 = C::value_reg(ctx, expr0_0);
+                            return Some(expr1_0);
+                        }
+                        _ => {}
+                    }
+                }
+                &InstructionData::Load {
+                    opcode: ref pattern5_0,
+                    arg: pattern5_1,
+                    flags: pattern5_2,
+                    offset: pattern5_3,
+                } => {
+                    match &pattern5_0 {
+                        &Opcode::Uload8 => {
+                            // Rule at src/isa/s390x/lower.isle line 1254.
+                            let expr0_0: Type = I8;
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr2_0 = constructor_zext64_mem(ctx, expr0_0, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Sload8 => {
+                            // Rule at src/isa/s390x/lower.isle line 1265.
+                            let expr0_0: Type = I8;
+                            let expr1_0 =
+                                constructor_lower_address(ctx, pattern5_2, pattern5_1, pattern5_3)?;
+                            let expr2_0 = constructor_sext64_mem(ctx, expr0_0, &expr1_0)?;
+                            let expr3_0 = C::value_reg(ctx, expr2_0);
+                            return Some(expr3_0);
+                        }
+                        &Opcode::Uload16 => {
+                            if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1288.
+                                let expr0_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr1_0 = constructor_loadrev16(ctx, &expr0_0)?;
+                                let expr2_0: Type = I16;
+                                let expr3_0 = constructor_zext64_reg(ctx, expr2_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1283.
+                                let expr0_0: Type = I16;
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr2_0 = constructor_zext64_mem(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                        }
+                        &Opcode::Sload16 => {
+                            if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1313.
+                                let expr0_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr1_0 = constructor_loadrev16(ctx, &expr0_0)?;
+                                let expr2_0: Type = I16;
+                                let expr3_0 = constructor_sext64_reg(ctx, expr2_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1308.
+                                let expr0_0: Type = I16;
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr2_0 = constructor_sext64_mem(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                        }
+                        &Opcode::Uload32 => {
+                            if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1327.
+                                let expr0_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr1_0 = constructor_loadrev32(ctx, &expr0_0)?;
+                                let expr2_0: Type = I32;
+                                let expr3_0 = constructor_zext64_reg(ctx, expr2_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1322.
+                                let expr0_0: Type = I32;
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr2_0 = constructor_zext64_mem(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                        }
+                        &Opcode::Sload32 => {
+                            if let Some(()) = C::littleendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1341.
+                                let expr0_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr1_0 = constructor_loadrev32(ctx, &expr0_0)?;
+                                let expr2_0: Type = I32;
+                                let expr3_0 = constructor_sext64_reg(ctx, expr2_0, expr1_0)?;
+                                let expr4_0 = C::value_reg(ctx, expr3_0);
+                                return Some(expr4_0);
+                            }
+                            if let Some(()) = C::bigendian(ctx, pattern5_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1336.
+                                let expr0_0: Type = I32;
+                                let expr1_0 = constructor_lower_address(
+                                    ctx, pattern5_2, pattern5_1, pattern5_3,
+                                )?;
+                                let expr2_0 = constructor_sext64_mem(ctx, expr0_0, &expr1_0)?;
+                                let expr3_0 = C::value_reg(ctx, expr2_0);
+                                return Some(expr3_0);
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term value_regs_ifcout.
+pub fn constructor_value_regs_ifcout<C: Context>(ctx: &mut C, arg0: Reg) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/lower.isle line 149.
+    let expr0_0: Type = I64;
+    let expr1_0 = C::temp_writable_reg(ctx, expr0_0);
+    let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0);
+    let expr3_0 = C::value_regs(ctx, pattern0_0, expr2_0);
+    return Some(expr3_0);
+}
+
+// Generated as internal constructor for term zero_divisor_check_needed.
+pub fn constructor_zero_divisor_check_needed<C: Context>(ctx: &mut C, arg0: Value) -> Option<bool> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::i64_from_value(ctx, pattern0_0) {
+        let pattern2_0 = 0;
+        if let Some(pattern3_0) = C::i64_nonequal(ctx, pattern1_0, pattern2_0) {
+            // Rule at src/isa/s390x/lower.isle line 339.
+            let expr0_0: bool = false;
+            return Some(expr0_0);
+        }
+    }
+    let pattern1_0 = C::value_type(ctx, pattern0_0);
+    if let Some(()) = C::allow_div_traps(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/lower.isle line 340.
+        let expr0_0: bool = false;
+        return Some(expr0_0);
+    }
+    // Rule at src/isa/s390x/lower.isle line 341.
+    let expr0_0: bool = true;
+    return Some(expr0_0);
+}
+
+// Generated as internal constructor for term maybe_trap_if_zero_divisor.
+pub fn constructor_maybe_trap_if_zero_divisor<C: Context>(
+    ctx: &mut C,
+    arg0: bool,
+    arg1: Type,
+    arg2: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == true {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/lower.isle line 347.
+        let expr0_0: i16 = 0;
+        let expr1_0 = IntCC::Equal;
+        let expr2_0 = C::intcc_as_cond(ctx, &expr1_0);
+        let expr3_0 = C::trap_code_division_by_zero(ctx);
+        let expr4_0 = constructor_icmps_simm16_and_trap(
+            ctx, pattern2_0, pattern3_0, expr0_0, &expr2_0, &expr3_0,
+        )?;
+        return Some(expr4_0);
+    }
+    if pattern0_0 == false {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        // Rule at src/isa/s390x/lower.isle line 346.
+        let expr0_0 = C::invalid_reg(ctx);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term div_overflow_check_needed.
+pub fn constructor_div_overflow_check_needed<C: Context>(ctx: &mut C, arg0: Value) -> Option<bool> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::i64_from_value(ctx, pattern0_0) {
+        let pattern2_0 = -1;
+        if let Some(pattern3_0) = C::i64_nonequal(ctx, pattern1_0, pattern2_0) {
+            // Rule at src/isa/s390x/lower.isle line 420.
+            let expr0_0: bool = false;
+            return Some(expr0_0);
+        }
+    }
+    // Rule at src/isa/s390x/lower.isle line 421.
+    let expr0_0: bool = true;
+    return Some(expr0_0);
+}
+
+// Generated as internal constructor for term maybe_trap_if_sdiv_overflow.
+pub fn constructor_maybe_trap_if_sdiv_overflow<C: Context>(
+    ctx: &mut C,
+    arg0: bool,
+    arg1: Type,
+    arg2: Type,
+    arg3: &RegPair,
+    arg4: Reg,
+) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == true {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        let pattern5_0 = arg4;
+        // Rule at src/isa/s390x/lower.isle line 434.
+        let expr0_0 = constructor_int_max(ctx, pattern3_0)?;
+        let expr1_0 = constructor_imm(ctx, pattern2_0, expr0_0)?;
+        let expr2_0 = constructor_regpair_lo(ctx, pattern4_0)?;
+        let expr3_0 = constructor_xor_reg(ctx, pattern2_0, expr1_0, expr2_0)?;
+        let expr4_0 = constructor_and_reg(ctx, pattern2_0, expr3_0, pattern5_0)?;
+        let expr5_0: i16 = -1;
+        let expr6_0 = IntCC::Equal;
+        let expr7_0 = C::intcc_as_cond(ctx, &expr6_0);
+        let expr8_0 = C::trap_code_integer_overflow(ctx);
+        let expr9_0 = constructor_icmps_simm16_and_trap(
+            ctx, pattern2_0, expr4_0, expr5_0, &expr7_0, &expr8_0,
+        )?;
+        return Some(expr9_0);
+    }
+    if pattern0_0 == false {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        let pattern5_0 = arg4;
+        // Rule at src/isa/s390x/lower.isle line 433.
+        let expr0_0 = C::invalid_reg(ctx);
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term int_max.
+pub fn constructor_int_max<C: Context>(ctx: &mut C, arg0: Type) -> Option<u64> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/lower.isle line 442.
+        let expr0_0: u64 = 127;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/lower.isle line 443.
+        let expr0_0: u64 = 32767;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/lower.isle line 444.
+        let expr0_0: u64 = 2147483647;
+        return Some(expr0_0);
+    }
+    if pattern0_0 == I64 {
+        // Rule at src/isa/s390x/lower.isle line 445.
+        let expr0_0: u64 = 9223372036854775807;
+        return Some(expr0_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term maybe_avoid_srem_overflow.
+pub fn constructor_maybe_avoid_srem_overflow<C: Context>(
+    ctx: &mut C,
+    arg0: bool,
+    arg1: Type,
+    arg2: &RegPair,
+    arg3: Reg,
+) -> Option<RegPair> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == true {
+        let pattern2_0 = arg1;
+        if pattern2_0 == I32 {
+            let pattern4_0 = arg2;
+            let pattern5_0 = arg3;
+            // Rule at src/isa/s390x/lower.isle line 463.
+            return Some(pattern4_0.clone());
+        }
+        if pattern2_0 == I64 {
+            let pattern4_0 = arg2;
+            let pattern5_0 = arg3;
+            // Rule at src/isa/s390x/lower.isle line 464.
+            let expr0_0: Type = I64;
+            let expr1_0: Type = I64;
+            let expr2_0: i16 = -1;
+            let expr3_0 = constructor_icmps_simm16(ctx, expr1_0, pattern5_0, expr2_0)?;
+            let expr4_0 = IntCC::Equal;
+            let expr5_0 = C::intcc_as_cond(ctx, &expr4_0);
+            let expr6_0: i16 = 0;
+            let expr7_0 = constructor_cmov_imm_regpair_lo(
+                ctx, expr0_0, &expr3_0, &expr5_0, expr6_0, pattern4_0,
+            )?;
+            return Some(expr7_0);
+        }
+    }
+    if pattern0_0 == false {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 462.
+        return Some(pattern3_0.clone());
+    }
+    return None;
+}
+
+// Generated as internal constructor for term cast_bool.
+pub fn constructor_cast_bool<C: Context>(ctx: &mut C, arg0: Type, arg1: Value) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == B1 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B1 {
+            // Rule at src/isa/s390x/lower.isle line 761.
+            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+            return Some(expr0_0);
+        }
+        if pattern3_0 == B8 {
+            // Rule at src/isa/s390x/lower.isle line 762.
+            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+            return Some(expr0_0);
+        }
+    }
+    if pattern0_0 == B8 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B8 {
+            // Rule at src/isa/s390x/lower.isle line 763.
+            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+            return Some(expr0_0);
+        }
+    }
+    if pattern0_0 == I8 {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B8 {
+            // Rule at src/isa/s390x/lower.isle line 764.
+            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+            return Some(expr0_0);
+        }
+    }
+    if let Some(pattern1_0) = C::fits_in_16(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B16 {
+            // Rule at src/isa/s390x/lower.isle line 765.
+            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+            return Some(expr0_0);
+        }
+    }
+    if let Some(pattern1_0) = C::fits_in_32(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B32 {
+            // Rule at src/isa/s390x/lower.isle line 766.
+            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+            return Some(expr0_0);
+        }
+    }
+    if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B64 {
+            // Rule at src/isa/s390x/lower.isle line 767.
+            let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+            return Some(expr0_0);
+        }
+    }
+    if let Some(pattern1_0) = C::gpr32_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B1 {
+            // Rule at src/isa/s390x/lower.isle line 770.
+            let expr0_0: Type = I32;
+            let expr1_0: Type = I32;
+            let expr2_0 = C::put_in_reg(ctx, pattern2_0);
+            let expr3_0: u8 = 31;
+            let expr4_0 = constructor_lshl_imm(ctx, expr1_0, expr2_0, expr3_0)?;
+            let expr5_0: u8 = 31;
+            let expr6_0 = constructor_ashr_imm(ctx, expr0_0, expr4_0, expr5_0)?;
+            return Some(expr6_0);
+        }
+        if pattern3_0 == B8 {
+            // Rule at src/isa/s390x/lower.isle line 776.
+            let expr0_0: Type = I8;
+            let expr1_0 = C::put_in_reg(ctx, pattern2_0);
+            let expr2_0 = constructor_sext32_reg(ctx, expr0_0, expr1_0)?;
+            return Some(expr2_0);
+        }
+        if pattern3_0 == B16 {
+            // Rule at src/isa/s390x/lower.isle line 778.
+            let expr0_0: Type = I16;
+            let expr1_0 = C::put_in_reg(ctx, pattern2_0);
+            let expr2_0 = constructor_sext32_reg(ctx, expr0_0, expr1_0)?;
+            return Some(expr2_0);
+        }
+    }
+    if let Some(pattern1_0) = C::gpr64_ty(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if pattern3_0 == B1 {
+            // Rule at src/isa/s390x/lower.isle line 772.
+            let expr0_0: Type = I64;
+            let expr1_0: Type = I64;
+            let expr2_0 = C::put_in_reg(ctx, pattern2_0);
+            let expr3_0: u8 = 63;
+            let expr4_0 = constructor_lshl_imm(ctx, expr1_0, expr2_0, expr3_0)?;
+            let expr5_0: u8 = 63;
+            let expr6_0 = constructor_ashr_imm(ctx, expr0_0, expr4_0, expr5_0)?;
+            return Some(expr6_0);
+        }
+        if pattern3_0 == B8 {
+            // Rule at src/isa/s390x/lower.isle line 780.
+            let expr0_0: Type = I8;
+            let expr1_0 = C::put_in_reg(ctx, pattern2_0);
+            let expr2_0 = constructor_sext64_reg(ctx, expr0_0, expr1_0)?;
+            return Some(expr2_0);
+        }
+        if pattern3_0 == B16 {
+            // Rule at src/isa/s390x/lower.isle line 782.
+            let expr0_0: Type = I16;
+            let expr1_0 = C::put_in_reg(ctx, pattern2_0);
+            let expr2_0 = constructor_sext64_reg(ctx, expr0_0, expr1_0)?;
+            return Some(expr2_0);
+        }
+        if pattern3_0 == B32 {
+            // Rule at src/isa/s390x/lower.isle line 784.
+            let expr0_0: Type = I32;
+            let expr1_0 = C::put_in_reg(ctx, pattern2_0);
+            let expr2_0 = constructor_sext64_reg(ctx, expr0_0, expr1_0)?;
+            return Some(expr2_0);
+        }
+    }
+    return None;
+}
+
+// Generated as internal constructor for term clz_offset.
+pub fn constructor_clz_offset<C: Context>(ctx: &mut C, arg0: Type, arg1: Reg) -> Option<Reg> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/lower.isle line 809.
+        let expr0_0: Type = I8;
+        let expr1_0: i16 = -56;
+        let expr2_0 = constructor_add_simm16(ctx, expr0_0, pattern2_0, expr1_0)?;
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I16 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/lower.isle line 810.
+        let expr0_0: Type = I16;
+        let expr1_0: i16 = -48;
+        let expr2_0 = constructor_add_simm16(ctx, expr0_0, pattern2_0, expr1_0)?;
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I32 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/lower.isle line 811.
+        let expr0_0: Type = I32;
+        let expr1_0: i16 = -32;
+        let expr2_0 = constructor_add_simm16(ctx, expr0_0, pattern2_0, expr1_0)?;
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I64 {
+        let pattern2_0 = arg1;
+        // Rule at src/isa/s390x/lower.isle line 812.
+        let expr0_0: Type = I64;
+        let expr1_0 = constructor_copy_reg(ctx, expr0_0, pattern2_0)?;
+        return Some(expr1_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term ctz_guardbit.
+pub fn constructor_ctz_guardbit<C: Context>(ctx: &mut C, arg0: Type) -> Option<UImm16Shifted> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == I8 {
+        // Rule at src/isa/s390x/lower.isle line 861.
+        let expr0_0: u16 = 256;
+        let expr1_0: u8 = 0;
+        let expr2_0 = C::uimm16shifted(ctx, expr0_0, expr1_0);
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I16 {
+        // Rule at src/isa/s390x/lower.isle line 862.
+        let expr0_0: u16 = 1;
+        let expr1_0: u8 = 16;
+        let expr2_0 = C::uimm16shifted(ctx, expr0_0, expr1_0);
+        return Some(expr2_0);
+    }
+    if pattern0_0 == I32 {
+        // Rule at src/isa/s390x/lower.isle line 863.
+        let expr0_0: u16 = 1;
+        let expr1_0: u8 = 32;
+        let expr2_0 = C::uimm16shifted(ctx, expr0_0, expr1_0);
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term istore8_impl.
+pub fn constructor_istore8_impl<C: Context>(
+    ctx: &mut C,
+    arg0: MemFlags,
+    arg1: Value,
+    arg2: Value,
+    arg3: Offset32,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if let Some(pattern2_0) = C::u8_from_value(ctx, pattern1_0) {
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1423.
+        let expr0_0 = constructor_lower_address(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr1_0 = constructor_store8_imm(ctx, pattern2_0, &expr0_0)?;
+        return Some(expr1_0);
+    }
+    let pattern2_0 = arg2;
+    let pattern3_0 = arg3;
+    // Rule at src/isa/s390x/lower.isle line 1419.
+    let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+    let expr1_0 = constructor_lower_address(ctx, pattern0_0, pattern2_0, pattern3_0)?;
+    let expr2_0 = constructor_store8(ctx, expr0_0, &expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term istore16_impl.
+pub fn constructor_istore16_impl<C: Context>(
+    ctx: &mut C,
+    arg0: MemFlags,
+    arg1: Value,
+    arg2: Value,
+    arg3: Offset32,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    if let Some(()) = C::littleendian(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        if let Some(pattern3_0) = C::i16_from_swapped_value(ctx, pattern2_0) {
+            let pattern4_0 = arg2;
+            let pattern5_0 = arg3;
+            // Rule at src/isa/s390x/lower.isle line 1449.
+            let expr0_0 = constructor_lower_address(ctx, pattern0_0, pattern4_0, pattern5_0)?;
+            let expr1_0 = constructor_store16_imm(ctx, pattern3_0, &expr0_0)?;
+            return Some(expr1_0);
+        }
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1441.
+        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+        let expr1_0 = constructor_lower_address(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr2_0 = constructor_storerev16(ctx, expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    if let Some(()) = C::bigendian(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        if let Some(pattern3_0) = C::i16_from_value(ctx, pattern2_0) {
+            let pattern4_0 = arg2;
+            let pattern5_0 = arg3;
+            // Rule at src/isa/s390x/lower.isle line 1445.
+            let expr0_0 = constructor_lower_address(ctx, pattern0_0, pattern4_0, pattern5_0)?;
+            let expr1_0 = constructor_store16_imm(ctx, pattern3_0, &expr0_0)?;
+            return Some(expr1_0);
+        }
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1437.
+        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+        let expr1_0 = constructor_lower_address(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr2_0 = constructor_store16(ctx, expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term istore32_impl.
+pub fn constructor_istore32_impl<C: Context>(
+    ctx: &mut C,
+    arg0: MemFlags,
+    arg1: Value,
+    arg2: Value,
+    arg3: Offset32,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    if let Some(()) = C::littleendian(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1471.
+        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+        let expr1_0 = constructor_lower_address(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr2_0 = constructor_storerev32(ctx, expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    if let Some(()) = C::bigendian(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        if let Some(pattern3_0) = C::i16_from_value(ctx, pattern2_0) {
+            let pattern4_0 = arg2;
+            let pattern5_0 = arg3;
+            // Rule at src/isa/s390x/lower.isle line 1467.
+            let expr0_0 = constructor_lower_address(ctx, pattern0_0, pattern4_0, pattern5_0)?;
+            let expr1_0 = constructor_store32_simm16(ctx, pattern3_0, &expr0_0)?;
+            return Some(expr1_0);
+        }
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1463.
+        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+        let expr1_0 = constructor_lower_address(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr2_0 = constructor_store32(ctx, expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term istore64_impl.
+pub fn constructor_istore64_impl<C: Context>(
+    ctx: &mut C,
+    arg0: MemFlags,
+    arg1: Value,
+    arg2: Value,
+    arg3: Offset32,
+) -> Option<SideEffectNoResult> {
+    let pattern0_0 = arg0;
+    if let Some(()) = C::littleendian(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1489.
+        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+        let expr1_0 = constructor_lower_address(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr2_0 = constructor_storerev64(ctx, expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    if let Some(()) = C::bigendian(ctx, pattern0_0) {
+        let pattern2_0 = arg1;
+        if let Some(pattern3_0) = C::i16_from_value(ctx, pattern2_0) {
+            let pattern4_0 = arg2;
+            let pattern5_0 = arg3;
+            // Rule at src/isa/s390x/lower.isle line 1485.
+            let expr0_0 = constructor_lower_address(ctx, pattern0_0, pattern4_0, pattern5_0)?;
+            let expr1_0 = constructor_store64_simm16(ctx, pattern3_0, &expr0_0)?;
+            return Some(expr1_0);
+        }
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1481.
+        let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+        let expr1_0 = constructor_lower_address(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr2_0 = constructor_store64(ctx, expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term atomic_store_impl.
+pub fn constructor_atomic_store_impl<C: Context>(
+    ctx: &mut C,
+    arg0: &SideEffectNoResult,
+) -> Option<ValueRegs> {
+    let pattern0_0 = arg0;
+    // Rule at src/isa/s390x/lower.isle line 1576.
+    let expr0_0 = constructor_value_regs_none(ctx, pattern0_0)?;
+    let expr1_0 = constructor_fence_impl(ctx)?;
+    let expr2_0 = constructor_value_regs_none(ctx, &expr1_0)?;
+    return Some(expr2_0);
+}
+
+// Generated as internal constructor for term icmp_val.
+pub fn constructor_icmp_val<C: Context>(
+    ctx: &mut C,
+    arg0: bool,
+    arg1: &IntCC,
+    arg2: Value,
+    arg3: Value,
+) -> Option<ProducesBool> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    if let Some(()) = C::signed(ctx, pattern1_0) {
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1643.
+        let expr0_0 = constructor_icmps_val(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr1_0 = C::intcc_as_cond(ctx, pattern1_0);
+        let expr2_0 = constructor_bool(ctx, &expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    if let Some(()) = C::unsigned(ctx, pattern1_0) {
+        let pattern3_0 = arg2;
+        let pattern4_0 = arg3;
+        // Rule at src/isa/s390x/lower.isle line 1646.
+        let expr0_0 = constructor_icmpu_val(ctx, pattern0_0, pattern3_0, pattern4_0)?;
+        let expr1_0 = C::intcc_as_cond(ctx, pattern1_0);
+        let expr2_0 = constructor_bool(ctx, &expr0_0, &expr1_0)?;
+        return Some(expr2_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term icmps_val.
+pub fn constructor_icmps_val<C: Context>(
+    ctx: &mut C,
+    arg0: bool,
+    arg1: Value,
+    arg2: Value,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == true {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if let Some(pattern4_0) = C::fits_in_64(ctx, pattern3_0) {
+            let pattern5_0 = arg2;
+            if let Some(pattern6_0) = C::sinkable_inst(ctx, pattern5_0) {
+                let pattern7_0 = C::inst_data(ctx, pattern6_0);
+                if let &InstructionData::Load {
+                    opcode: ref pattern8_0,
+                    arg: pattern8_1,
+                    flags: pattern8_2,
+                    offset: pattern8_3,
+                } = &pattern7_0
+                {
+                    match &pattern8_0 {
+                        &Opcode::Sload16 => {
+                            if let Some(()) = C::bigendian(ctx, pattern8_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1676.
+                                let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                                let expr1_0 = constructor_sink_sload16(ctx, pattern6_0)?;
+                                let expr2_0 = constructor_icmps_mem_sext16(
+                                    ctx, pattern4_0, expr0_0, &expr1_0,
+                                )?;
+                                return Some(expr2_0);
+                            }
+                        }
+                        &Opcode::Sload32 => {
+                            if let Some(()) = C::bigendian(ctx, pattern8_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1678.
+                                let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                                let expr1_0 = constructor_sink_sload32(ctx, pattern6_0)?;
+                                let expr2_0 = constructor_icmps_mem_sext32(
+                                    ctx, pattern4_0, expr0_0, &expr1_0,
+                                )?;
+                                return Some(expr2_0);
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            let pattern6_0 = C::value_type(ctx, pattern5_0);
+            if pattern6_0 == I16 {
+                if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern5_0) {
+                    let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                    if let &InstructionData::Load {
+                        opcode: ref pattern10_0,
+                        arg: pattern10_1,
+                        flags: pattern10_2,
+                        offset: pattern10_3,
+                    } = &pattern9_0
+                    {
+                        if let &Opcode::Load = &pattern10_0 {
+                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1672.
+                                let expr0_0 = constructor_ty_ext32(ctx, pattern4_0)?;
+                                let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern2_0)?;
+                                let expr2_0 = constructor_sink_load(ctx, pattern8_0)?;
+                                let expr3_0 =
+                                    constructor_icmps_mem_sext16(ctx, expr0_0, expr1_0, &expr2_0)?;
+                                return Some(expr3_0);
+                            }
+                        }
+                    }
+                }
+            }
+            if let Some(pattern7_0) = C::ty_32_or_64(ctx, pattern6_0) {
+                if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern5_0) {
+                    let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                    if let &InstructionData::Load {
+                        opcode: ref pattern10_0,
+                        arg: pattern10_1,
+                        flags: pattern10_2,
+                        offset: pattern10_3,
+                    } = &pattern9_0
+                    {
+                        if let &Opcode::Load = &pattern10_0 {
+                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1668.
+                                let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                                let expr1_0 = constructor_sink_load(ctx, pattern8_0)?;
+                                let expr2_0 =
+                                    constructor_icmps_mem(ctx, pattern4_0, expr0_0, &expr1_0)?;
+                                return Some(expr2_0);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    let pattern1_0 = arg1;
+    let pattern2_0 = C::value_type(ctx, pattern1_0);
+    if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) {
+        let pattern4_0 = arg2;
+        if let Some(pattern5_0) = C::i16_from_value(ctx, pattern4_0) {
+            // Rule at src/isa/s390x/lower.isle line 1662.
+            let expr0_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+            let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern1_0)?;
+            let expr2_0 = constructor_icmps_simm16(ctx, expr0_0, expr1_0, pattern5_0)?;
+            return Some(expr2_0);
+        }
+        if let Some(pattern5_0) = C::i32_from_value(ctx, pattern4_0) {
+            // Rule at src/isa/s390x/lower.isle line 1664.
+            let expr0_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+            let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern1_0)?;
+            let expr2_0 = constructor_icmps_simm32(ctx, expr0_0, expr1_0, pattern5_0)?;
+            return Some(expr2_0);
+        }
+        if let Some(pattern5_0) = C::def_inst(ctx, pattern4_0) {
+            let pattern6_0 = C::inst_data(ctx, pattern5_0);
+            if let &InstructionData::Unary {
+                opcode: ref pattern7_0,
+                arg: pattern7_1,
+            } = &pattern6_0
+            {
+                if let &Opcode::Sextend = &pattern7_0 {
+                    let pattern9_0 = C::value_type(ctx, pattern7_1);
+                    if pattern9_0 == I32 {
+                        // Rule at src/isa/s390x/lower.isle line 1658.
+                        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                        let expr2_0 =
+                            constructor_icmps_reg_sext32(ctx, pattern3_0, expr0_0, expr1_0)?;
+                        return Some(expr2_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/lower.isle line 1654.
+        let expr0_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+        let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern1_0)?;
+        let expr2_0 = constructor_put_in_reg_sext32(ctx, pattern4_0)?;
+        let expr3_0 = constructor_icmps_reg(ctx, expr0_0, expr1_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term icmpu_val.
+pub fn constructor_icmpu_val<C: Context>(
+    ctx: &mut C,
+    arg0: bool,
+    arg1: Value,
+    arg2: Value,
+) -> Option<ProducesFlags> {
+    let pattern0_0 = arg0;
+    if pattern0_0 == true {
+        let pattern2_0 = arg1;
+        let pattern3_0 = C::value_type(ctx, pattern2_0);
+        if let Some(pattern4_0) = C::fits_in_64(ctx, pattern3_0) {
+            let pattern5_0 = arg2;
+            if let Some(pattern6_0) = C::sinkable_inst(ctx, pattern5_0) {
+                let pattern7_0 = C::inst_data(ctx, pattern6_0);
+                if let &InstructionData::Load {
+                    opcode: ref pattern8_0,
+                    arg: pattern8_1,
+                    flags: pattern8_2,
+                    offset: pattern8_3,
+                } = &pattern7_0
+                {
+                    match &pattern8_0 {
+                        &Opcode::Uload16 => {
+                            if let Some(pattern10_0) = C::def_inst(ctx, pattern8_1) {
+                                if let Some((pattern11_0, pattern11_1, pattern11_2)) =
+                                    C::symbol_value_data(ctx, pattern10_0)
+                                {
+                                    if let Some(()) = C::reloc_distance_near(ctx, &pattern11_1) {
+                                        let pattern13_0 = C::i64_from_offset(ctx, pattern8_3);
+                                        let closure14 = || {
+                                            return Some(pattern11_2);
+                                        };
+                                        if let Some(pattern14_0) = closure14() {
+                                            if let Some(pattern15_0) = C::memarg_symbol_offset_sum(
+                                                ctx,
+                                                pattern13_0,
+                                                pattern14_0,
+                                            ) {
+                                                let pattern16_0 = C::inst_data(ctx, pattern6_0);
+                                                if let &InstructionData::Load {
+                                                    opcode: ref pattern17_0,
+                                                    arg: pattern17_1,
+                                                    flags: pattern17_2,
+                                                    offset: pattern17_3,
+                                                } = &pattern16_0
+                                                {
+                                                    if let &Opcode::Uload16 = &pattern17_0 {
+                                                        if let Some(()) =
+                                                            C::bigendian(ctx, pattern17_2)
+                                                        {
+                                                            // Rule at src/isa/s390x/lower.isle line 1711.
+                                                            let expr0_0 =
+                                                                C::put_in_reg(ctx, pattern2_0);
+                                                            let expr1_0 = constructor_sink_uload16(
+                                                                ctx, pattern6_0,
+                                                            )?;
+                                                            let expr2_0 =
+                                                                constructor_icmpu_mem_zext16(
+                                                                    ctx, pattern4_0, expr0_0,
+                                                                    &expr1_0,
+                                                                )?;
+                                                            return Some(expr2_0);
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        &Opcode::Uload32 => {
+                            if let Some(()) = C::bigendian(ctx, pattern8_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1714.
+                                let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                                let expr1_0 = constructor_sink_uload32(ctx, pattern6_0)?;
+                                let expr2_0 = constructor_icmpu_mem_zext32(
+                                    ctx, pattern4_0, expr0_0, &expr1_0,
+                                )?;
+                                return Some(expr2_0);
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            let pattern6_0 = C::value_type(ctx, pattern5_0);
+            if pattern6_0 == I16 {
+                if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern5_0) {
+                    let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                    if let &InstructionData::Load {
+                        opcode: ref pattern10_0,
+                        arg: pattern10_1,
+                        flags: pattern10_2,
+                        offset: pattern10_3,
+                    } = &pattern9_0
+                    {
+                        if let &Opcode::Load = &pattern10_0 {
+                            if let Some(pattern12_0) = C::def_inst(ctx, pattern10_1) {
+                                if let Some((pattern13_0, pattern13_1, pattern13_2)) =
+                                    C::symbol_value_data(ctx, pattern12_0)
+                                {
+                                    if let Some(()) = C::reloc_distance_near(ctx, &pattern13_1) {
+                                        let pattern15_0 = C::i64_from_offset(ctx, pattern10_3);
+                                        let closure16 = || {
+                                            return Some(pattern13_2);
+                                        };
+                                        if let Some(pattern16_0) = closure16() {
+                                            if let Some(pattern17_0) = C::memarg_symbol_offset_sum(
+                                                ctx,
+                                                pattern15_0,
+                                                pattern16_0,
+                                            ) {
+                                                let pattern18_0 = C::inst_data(ctx, pattern8_0);
+                                                if let &InstructionData::Load {
+                                                    opcode: ref pattern19_0,
+                                                    arg: pattern19_1,
+                                                    flags: pattern19_2,
+                                                    offset: pattern19_3,
+                                                } = &pattern18_0
+                                                {
+                                                    if let &Opcode::Load = &pattern19_0 {
+                                                        if let Some(()) =
+                                                            C::bigendian(ctx, pattern19_2)
+                                                        {
+                                                            // Rule at src/isa/s390x/lower.isle line 1704.
+                                                            let expr0_0 = constructor_ty_ext32(
+                                                                ctx, pattern4_0,
+                                                            )?;
+                                                            let expr1_0 =
+                                                                constructor_put_in_reg_zext32(
+                                                                    ctx, pattern2_0,
+                                                                )?;
+                                                            let expr2_0 = constructor_sink_load(
+                                                                ctx, pattern8_0,
+                                                            )?;
+                                                            let expr3_0 =
+                                                                constructor_icmpu_mem_zext16(
+                                                                    ctx, expr0_0, expr1_0, &expr2_0,
+                                                                )?;
+                                                            return Some(expr3_0);
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if let Some(pattern7_0) = C::ty_32_or_64(ctx, pattern6_0) {
+                if let Some(pattern8_0) = C::sinkable_inst(ctx, pattern5_0) {
+                    let pattern9_0 = C::inst_data(ctx, pattern8_0);
+                    if let &InstructionData::Load {
+                        opcode: ref pattern10_0,
+                        arg: pattern10_1,
+                        flags: pattern10_2,
+                        offset: pattern10_3,
+                    } = &pattern9_0
+                    {
+                        if let &Opcode::Load = &pattern10_0 {
+                            if let Some(()) = C::bigendian(ctx, pattern10_2) {
+                                // Rule at src/isa/s390x/lower.isle line 1698.
+                                let expr0_0 = C::put_in_reg(ctx, pattern2_0);
+                                let expr1_0 = constructor_sink_load(ctx, pattern8_0)?;
+                                let expr2_0 =
+                                    constructor_icmpu_mem(ctx, pattern4_0, expr0_0, &expr1_0)?;
+                                return Some(expr2_0);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    let pattern1_0 = arg1;
+    let pattern2_0 = C::value_type(ctx, pattern1_0);
+    if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) {
+        let pattern4_0 = arg2;
+        if let Some(pattern5_0) = C::u32_from_value(ctx, pattern4_0) {
+            // Rule at src/isa/s390x/lower.isle line 1694.
+            let expr0_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+            let expr1_0 = constructor_put_in_reg_zext32(ctx, pattern1_0)?;
+            let expr2_0 = constructor_icmpu_uimm32(ctx, expr0_0, expr1_0, pattern5_0)?;
+            return Some(expr2_0);
+        }
+        if let Some(pattern5_0) = C::def_inst(ctx, pattern4_0) {
+            let pattern6_0 = C::inst_data(ctx, pattern5_0);
+            if let &InstructionData::Unary {
+                opcode: ref pattern7_0,
+                arg: pattern7_1,
+            } = &pattern6_0
+            {
+                if let &Opcode::Uextend = &pattern7_0 {
+                    let pattern9_0 = C::value_type(ctx, pattern7_1);
+                    if pattern9_0 == I32 {
+                        // Rule at src/isa/s390x/lower.isle line 1690.
+                        let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+                        let expr1_0 = C::put_in_reg(ctx, pattern7_1);
+                        let expr2_0 =
+                            constructor_icmpu_reg_zext32(ctx, pattern3_0, expr0_0, expr1_0)?;
+                        return Some(expr2_0);
+                    }
+                }
+            }
+        }
+        // Rule at src/isa/s390x/lower.isle line 1686.
+        let expr0_0 = constructor_ty_ext32(ctx, pattern3_0)?;
+        let expr1_0 = constructor_put_in_reg_zext32(ctx, pattern1_0)?;
+        let expr2_0 = constructor_put_in_reg_zext32(ctx, pattern4_0)?;
+        let expr3_0 = constructor_icmpu_reg(ctx, expr0_0, expr1_0, expr2_0)?;
+        return Some(expr3_0);
+    }
+    return None;
+}
+
+// Generated as internal constructor for term fcmp_val.
+pub fn constructor_fcmp_val<C: Context>(
+    ctx: &mut C,
+    arg0: &FloatCC,
+    arg1: Value,
+    arg2: Value,
+) -> Option<ProducesBool> {
+    let pattern0_0 = arg0;
+    let pattern1_0 = arg1;
+    let pattern2_0 = C::value_type(ctx, pattern1_0);
+    let pattern3_0 = arg2;
+    // Rule at src/isa/s390x/lower.isle line 1727.
+    let expr0_0 = C::put_in_reg(ctx, pattern1_0);
+    let expr1_0 = C::put_in_reg(ctx, pattern3_0);
+    let expr2_0 = constructor_fcmp_reg(ctx, pattern2_0, expr0_0, expr1_0)?;
+    let expr3_0 = C::floatcc_as_cond(ctx, pattern0_0);
+    let expr4_0 = constructor_bool(ctx, &expr2_0, &expr3_0)?;
+    return Some(expr4_0);
+}
+
+// Generated as internal constructor for term value_nonzero.
+pub fn constructor_value_nonzero<C: Context>(ctx: &mut C, arg0: Value) -> Option<ProducesBool> {
+    let pattern0_0 = arg0;
+    if let Some(pattern1_0) = C::def_inst(ctx, pattern0_0) {
+        let pattern2_0 = C::inst_data(ctx, pattern1_0);
+        match &pattern2_0 {
+            &InstructionData::FloatCompare {
+                opcode: ref pattern3_0,
+                args: ref pattern3_1,
+                cond: ref pattern3_2,
+            } => {
+                if let &Opcode::Fcmp = &pattern3_0 {
+                    let (pattern5_0, pattern5_1) = C::unpack_value_array_2(ctx, &pattern3_1);
+                    // Rule at src/isa/s390x/lower.isle line 1755.
+                    let expr0_0 = constructor_fcmp_val(ctx, &pattern3_2, pattern5_0, pattern5_1)?;
+                    return Some(expr0_0);
+                }
+            }
+            &InstructionData::IntCompare {
+                opcode: ref pattern3_0,
+                args: ref pattern3_1,
+                cond: ref pattern3_2,
+            } => {
+                if let &Opcode::Icmp = &pattern3_0 {
+                    let (pattern5_0, pattern5_1) = C::unpack_value_array_2(ctx, &pattern3_1);
+                    // Rule at src/isa/s390x/lower.isle line 1754.
+                    let expr0_0: bool = false;
+                    let expr1_0 =
+                        constructor_icmp_val(ctx, expr0_0, &pattern3_2, pattern5_0, pattern5_1)?;
+                    return Some(expr1_0);
+                }
+            }
+            &InstructionData::Unary {
+                opcode: ref pattern3_0,
+                arg: pattern3_1,
+            } => {
+                if let &Opcode::Bint = &pattern3_0 {
+                    // Rule at src/isa/s390x/lower.isle line 1753.
+                    let expr0_0 = constructor_value_nonzero(ctx, pattern3_1)?;
+                    return Some(expr0_0);
+                }
+            }
+            _ => {}
+        }
+    }
+    let pattern1_0 = C::value_type(ctx, pattern0_0);
+    if let Some(pattern2_0) = C::gpr32_ty(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/lower.isle line 1756.
+        let expr0_0: Type = I32;
+        let expr1_0 = constructor_put_in_reg_sext32(ctx, pattern0_0)?;
+        let expr2_0: i16 = 0;
+        let expr3_0 = constructor_icmps_simm16(ctx, expr0_0, expr1_0, expr2_0)?;
+        let expr4_0 = IntCC::NotEqual;
+        let expr5_0 = C::intcc_as_cond(ctx, &expr4_0);
+        let expr6_0 = constructor_bool(ctx, &expr3_0, &expr5_0)?;
+        return Some(expr6_0);
+    }
+    if let Some(pattern2_0) = C::gpr64_ty(ctx, pattern1_0) {
+        // Rule at src/isa/s390x/lower.isle line 1759.
+        let expr0_0: Type = I64;
+        let expr1_0 = C::put_in_reg(ctx, pattern0_0);
+        let expr2_0: i16 = 0;
+        let expr3_0 = constructor_icmps_simm16(ctx, expr0_0, expr1_0, expr2_0)?;
+        let expr4_0 = IntCC::NotEqual;
+        let expr5_0 = C::intcc_as_cond(ctx, &expr4_0);
+        let expr6_0 = constructor_bool(ctx, &expr3_0, &expr5_0)?;
+        return Some(expr6_0);
+    }
+    return None;
+}
diff --git a/cranelift/filetests/filetests/isa/s390x/bitops.clif b/cranelift/filetests/filetests/isa/s390x/bitops.clif
index a6bb3308d0a7..8939a946af65 100644
--- a/cranelift/filetests/filetests/isa/s390x/bitops.clif
+++ b/cranelift/filetests/filetests/isa/s390x/bitops.clif
@@ -54,8 +54,7 @@ block0(v0: i32):
 
 ; check:  llgfr %r2, %r2
 ; nextln: flogr %r0, %r2
-; nextln: lr %r2, %r0
-; nextln: ahi %r2, -32
+; nextln: ahik %r2, %r0, -32
 ; nextln: br %r14
 
 function %clz_i16(i16) -> i16 {
@@ -66,8 +65,7 @@ block0(v0: i16):
 
 ; check:  llghr %r2, %r2
 ; nextln: flogr %r0, %r2
-; nextln: lr %r2, %r0
-; nextln: ahi %r2, -48
+; nextln: ahik %r2, %r0, -48
 ; nextln: br %r14
 
 function %clz_i8(i8) -> i8 {
@@ -78,8 +76,7 @@ block0(v0: i8):
 
 ; check:  llgcr %r2, %r2
 ; nextln: flogr %r0, %r2
-; nextln: lr %r2, %r0
-; nextln: ahi %r2, -56
+; nextln: ahik %r2, %r0, -56
 ; nextln: br %r14
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -93,7 +90,7 @@ block0(v0: i64):
 }
 
 ; check:  srag %r3, %r2, 63
-; nextln: xgrk %r2, %r3, %r2
+; nextln: xgr %r2, %r3
 ; nextln: flogr %r0, %r2
 ; nextln: lgr %r2, %r0
 ; nextln: br %r14
@@ -106,10 +103,9 @@ block0(v0: i32):
 
 ; check:  lgfr %r2, %r2
 ; nextln: srag %r3, %r2, 63
-; nextln: xgrk %r2, %r3, %r2
+; nextln: xgr %r2, %r3
 ; nextln: flogr %r0, %r2
-; nextln: lr %r2, %r0
-; nextln: ahi %r2, -32
+; nextln: ahik %r2, %r0, -32
 ; nextln: br %r14
 
 function %cls_i16(i16) -> i16 {
@@ -120,10 +116,9 @@ block0(v0: i16):
 
 ; check:  lghr %r2, %r2
 ; nextln: srag %r3, %r2, 63
-; nextln: xgrk %r2, %r3, %r2
+; nextln: xgr %r2, %r3
 ; nextln: flogr %r0, %r2
-; nextln: lr %r2, %r0
-; nextln: ahi %r2, -48
+; nextln: ahik %r2, %r0, -48
 ; nextln: br %r14
 
 function %cls_i8(i8) -> i8 {
@@ -134,10 +129,9 @@ block0(v0: i8):
 
 ; check:  lgbr %r2, %r2
 ; nextln: srag %r3, %r2, 63
-; nextln: xgrk %r2, %r3, %r2
+; nextln: xgr %r2, %r3
 ; nextln: flogr %r0, %r2
-; nextln: lr %r2, %r0
-; nextln: ahi %r2, -56
+; nextln: ahik %r2, %r0, -56
 ; nextln: br %r14
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -151,7 +145,7 @@ block0(v0: i64):
 }
 
 ; check:  lcgr %r3, %r2
-; nextln: ngrk %r2, %r3, %r2
+; nextln: ngr %r2, %r3
 ; nextln: flogr %r0, %r2
 ; nextln: locghie %r0, -1
 ; nextln: lghi %r2, 63
@@ -166,7 +160,7 @@ block0(v0: i32):
 
 ; check:  oihl %r2, 1
 ; nextln: lcgr %r3, %r2
-; nextln: ngrk %r2, %r3, %r2
+; nextln: ngr %r2, %r3
 ; nextln: flogr %r0, %r2
 ; nextln: lhi %r2, 63
 ; nextln: sr %r2, %r0
@@ -180,7 +174,7 @@ block0(v0: i16):
 
 ; check:  oilh %r2, 1
 ; nextln: lcgr %r3, %r2
-; nextln: ngrk %r2, %r3, %r2
+; nextln: ngr %r2, %r3
 ; nextln: flogr %r0, %r2
 ; nextln: lhi %r2, 63
 ; nextln: sr %r2, %r0
@@ -194,7 +188,7 @@ block0(v0: i8):
 
 ; check:  oill %r2, 256
 ; nextln: lcgr %r3, %r2
-; nextln: ngrk %r2, %r3, %r2
+; nextln: ngr %r2, %r3
 ; nextln: flogr %r0, %r2
 ; nextln: lhi %r2, 63
 ; nextln: sr %r2, %r0
diff --git a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
index 1eda3470cdd6..c07d5f66a75b 100644
--- a/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
+++ b/cranelift/filetests/filetests/isa/s390x/heap_addr.clif
@@ -13,8 +13,8 @@ block0(v0: i64, v1: i32):
 
 ; check: Block 0:
 ; check:  llgfr %r3, %r3
-; nextln: lg %r4, 0(%r2)
-; nextln: aghi %r4, 0
+; nextln: lghi %r4, 0
+; nextln: ag %r4, 0(%r2)
 ; nextln: clgr %r3, %r4
 ; nextln: jgnh label1 ; jg label2
 ; check: Block 1:
diff --git a/cranelift/filetests/filetests/isa/s390x/shift-rotate.clif b/cranelift/filetests/filetests/isa/s390x/shift-rotate.clif
index 9c682fe77842..0066bef49832 100644
--- a/cranelift/filetests/filetests/isa/s390x/shift-rotate.clif
+++ b/cranelift/filetests/filetests/isa/s390x/shift-rotate.clif
@@ -52,13 +52,12 @@ block0(v0: i16, v1: i16):
 }
 
 ; check:  llhr %r2, %r2
-; nextln: lr %r3, %r4
-; nextln: lcr %r4, %r4
+; nextln: lcr %r4, %r3
 ; nextln: nill %r3, 15
 ; nextln: nill %r4, 15
-; nextln: sllk %r3, %r2, 0(%r3)
-; nextln: srlk %r2, %r2, 0(%r4)
-; nextln: ork %r2, %r3, %r2
+; nextln: sllk %r4, %r2, 0(%r4)
+; nextln: srlk %r2, %r2, 0(%r3)
+; nextln: ork %r2, %r4, %r2
 ; nextln: br %r14
 
 function %rotr_i16_imm(i16) -> i16 {
@@ -81,13 +80,12 @@ block0(v0: i8, v1: i8):
 }
 
 ; check:  llcr %r2, %r2
-; nextln: lr %r3, %r4
-; nextln: lcr %r4, %r4
+; nextln: lcr %r4, %r3
 ; nextln: nill %r3, 7
 ; nextln: nill %r4, 7
-; nextln: sllk %r3, %r2, 0(%r3)
-; nextln: srlk %r2, %r2, 0(%r4)
-; nextln: ork %r2, %r3, %r2
+; nextln: sllk %r4, %r2, 0(%r4)
+; nextln: srlk %r2, %r2, 0(%r3)
+; nextln: ork %r2, %r4, %r2
 ; nextln: br %r14
 
 function %rotr_i8_imm(i8) -> i8 {
@@ -152,10 +150,9 @@ block0(v0: i16, v1: i16):
 }
 
 ; check:  llhr %r2, %r2
-; nextln: lr %r4, %r3
-; nextln: lcr %r3, %r3
-; nextln: nill %r4, 15
+; nextln: lcr %r4, %r3
 ; nextln: nill %r3, 15
+; nextln: nill %r4, 15
 ; nextln: sllk %r3, %r2, 0(%r3)
 ; nextln: srlk %r2, %r2, 0(%r4)
 ; nextln: ork %r2, %r3, %r2
@@ -181,10 +178,9 @@ block0(v0: i8, v1: i8):
 }
 
 ; check:  llcr %r2, %r2
-; nextln: lr %r4, %r3
-; nextln: lcr %r3, %r3
-; nextln: nill %r4, 7
+; nextln: lcr %r4, %r3
 ; nextln: nill %r3, 7
+; nextln: nill %r4, 7
 ; nextln: sllk %r3, %r2, 0(%r3)
 ; nextln: srlk %r2, %r2, 0(%r4)
 ; nextln: ork %r2, %r3, %r2